Hugging Face's logo

it-support-mumz
/
Arabic-TTS-Spark

Text-to-Speech
Safetensors
Arabic
spark-tts
speech
arabic
spark
tts
custom_code
Model card Files
xet
 Community
it-support-mumz commited on
Commit
a6afb46
·
verified ·
1 Parent(s): 565139a

Fork Arabic-TTS-Spark with HF Inference Endpoints handler

Browse files
Files changed (39) hide show
  1. .gitattributes +11 -0
  2. BiCodec/config.yaml +60 -0
  3. BiCodec/model.safetensors +3 -0
  4. LLM/added_tokens.json +0 -0
  5. LLM/config.json +27 -0
  6. LLM/generation_config.json +8 -0
  7. LLM/model.safetensors +3 -0
  8. LLM/special_tokens_map.json +31 -0
  9. LLM/tokenizer.json +3 -0
  10. LLM/tokenizer_config.json +0 -0
  11. LLM/trainer_state.json +3964 -0
  12. LLM/vocab.json +0 -0
  13. README.md +138 -0
  14. config.json +83 -0
  15. config.yaml +7 -0
  16. configuration_spark_tts.py +233 -0
  17. handler.py +296 -0
  18. modeling_spark_tts.py +0 -0
  19. processing_spark_tts.py +889 -0
  20. reference.wav +3 -0
  21. requirements.txt +23 -0
  22. src/figures/gradio_TTS.png +0 -0
  23. src/figures/gradio_control.png +0 -0
  24. src/figures/infer_control.png +3 -0
  25. src/figures/infer_voice_cloning.png +3 -0
  26. src/logo/HKUST.jpg +3 -0
  27. src/logo/NPU.jpg +3 -0
  28. src/logo/NTU.jpg +0 -0
  29. src/logo/SJU.jpg +3 -0
  30. src/logo/SparkAudio.jpg +0 -0
  31. src/logo/SparkAudio2.jpg +0 -0
  32. src/logo/SparkTTS.jpg +0 -0
  33. src/logo/SparkTTS.png +3 -0
  34. src/logo/mobvoi.jpg +3 -0
  35. src/logo/mobvoi.png +3 -0
  36. wav2vec2-large-xlsr-53/README.md +29 -0
  37. wav2vec2-large-xlsr-53/config.json +83 -0
  38. wav2vec2-large-xlsr-53/preprocessor_config.json +9 -0
  39. wav2vec2-large-xlsr-53/pytorch_model.bin +3 -0
.gitattributes CHANGED
@@ -33,3 +33,14 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ LLM/tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
+ pretrained_models/Spark-TTS-0.5B/LLM/tokenizer.json filter=lfs diff=lfs merge=lfs -text
38
+ src/figures/infer_control.png filter=lfs diff=lfs merge=lfs -text
39
+ src/figures/infer_voice_cloning.png filter=lfs diff=lfs merge=lfs -text
40
+ src/logo/HKUST.jpg filter=lfs diff=lfs merge=lfs -text
41
+ src/logo/NPU.jpg filter=lfs diff=lfs merge=lfs -text
42
+ src/logo/SJU.jpg filter=lfs diff=lfs merge=lfs -text
43
+ src/logo/SparkTTS.png filter=lfs diff=lfs merge=lfs -text
44
+ src/logo/mobvoi.jpg filter=lfs diff=lfs merge=lfs -text
45
+ src/logo/mobvoi.png filter=lfs diff=lfs merge=lfs -text
46
+ reference.wav filter=lfs diff=lfs merge=lfs -text
BiCodec/config.yaml ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ audio_tokenizer:
2
+ mel_params:
3
+ sample_rate: 16000
4
+ n_fft: 1024
5
+ win_length: 640
6
+ hop_length: 320
7
+ mel_fmin: 10
8
+ mel_fmax: null
9
+ num_mels: 128
10
+
11
+ encoder:
12
+ input_channels: 1024
13
+ vocos_dim: 384
14
+ vocos_intermediate_dim: 2048
15
+ vocos_num_layers: 12
16
+ out_channels: 1024
17
+ sample_ratios: [1,1]
18
+
19
+ decoder:
20
+ input_channel: 1024
21
+ channels: 1536
22
+ rates: [8, 5, 4, 2]
23
+ kernel_sizes: [16,11,8,4]
24
+
25
+ quantizer:
26
+ input_dim: 1024
27
+ codebook_size: 8192
28
+ codebook_dim: 8
29
+ commitment: 0.25
30
+ codebook_loss_weight: 2.0
31
+ use_l2_normlize: True
32
+ threshold_ema_dead_code: 0.2
33
+
34
+ speaker_encoder:
35
+ input_dim: 128
36
+ out_dim: 1024
37
+ latent_dim: 128
38
+ token_num: 32
39
+ fsq_levels: [4, 4, 4, 4, 4, 4]
40
+ fsq_num_quantizers: 1
41
+
42
+ prenet:
43
+ input_channels: 1024
44
+ vocos_dim: 384
45
+ vocos_intermediate_dim: 2048
46
+ vocos_num_layers: 12
47
+ out_channels: 1024
48
+ condition_dim: 1024
49
+ sample_ratios: [1,1]
50
+ use_tanh_at_final: False
51
+
52
+ postnet:
53
+ input_channels: 1024
54
+ vocos_dim: 384
55
+ vocos_intermediate_dim: 2048
56
+ vocos_num_layers: 6
57
+ out_channels: 1024
58
+ use_tanh_at_final: False
59
+
60
+
BiCodec/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e9940cd48d4446e4340ced82d234bf5618350dd9f5db900ebe47a4fdb03867ec
3
+ size 625518756
LLM/added_tokens.json ADDED
The diff for this file is too large to render. See raw diff
 
LLM/config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "eos_token_id": 151645,
7
+ "hidden_act": "silu",
8
+ "hidden_size": 896,
9
+ "initializer_range": 0.02,
10
+ "intermediate_size": 4864,
11
+ "max_position_embeddings": 32768,
12
+ "max_window_layers": 21,
13
+ "model_type": "qwen2",
14
+ "num_attention_heads": 14,
15
+ "num_hidden_layers": 24,
16
+ "num_key_value_heads": 2,
17
+ "rms_norm_eps": 1e-06,
18
+ "rope_scaling": null,
19
+ "rope_theta": 1000000.0,
20
+ "sliding_window": 32768,
21
+ "tie_word_embeddings": true,
22
+ "torch_dtype": "bfloat16",
23
+ "transformers_version": "4.51.3",
24
+ "use_cache": false,
25
+ "use_sliding_window": false,
26
+ "vocab_size": 166000
27
+ }
LLM/generation_config.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 151643,
4
+ "do_sample": true,
5
+ "eos_token_id": 151645,
6
+ "transformers_version": "4.51.3",
7
+ "use_cache": false
8
+ }
LLM/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5d7e61a5ed6a6dbfc8cd485c7b8536142670547da447c48fe6e28210b8fd3715
3
+ size 1013300536
LLM/special_tokens_map.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|object_ref_start|>",
6
+ "<|object_ref_end|>",
7
+ "<|box_start|>",
8
+ "<|box_end|>",
9
+ "<|quad_start|>",
10
+ "<|quad_end|>",
11
+ "<|vision_start|>",
12
+ "<|vision_end|>",
13
+ "<|vision_pad|>",
14
+ "<|image_pad|>",
15
+ "<|video_pad|>"
16
+ ],
17
+ "eos_token": {
18
+ "content": "<|im_end|>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ "pad_token": {
25
+ "content": "<|endoftext|>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ }
31
+ }
LLM/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c8b057d6ca205a429cc3428b9fc815f0d6ee1d53106dd5e5b129ef9db2ff057
3
+ size 14129172
LLM/tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff
 
LLM/trainer_state.json ADDED
@@ -0,0 +1,3964 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 44.270833333333336,
6
+ "eval_steps": 576,
7
+ "global_step": 25500,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.001736111111111111,
14
+ "eval_loss": 9.594602584838867,
15
+ "eval_runtime": 41.3373,
16
+ "eval_samples_per_second": 90.209,
17
+ "eval_steps_per_second": 5.661,
18
+ "step": 1
19
+ },
20
+ {
21
+ "epoch": 0.08680555555555555,
22
+ "grad_norm": 12.75,
23
+ "learning_rate": 0.000196,
24
+ "loss": 7.4156,
25
+ "step": 50
26
+ },
27
+ {
28
+ "epoch": 0.1736111111111111,
29
+ "grad_norm": 13.3125,
30
+ "learning_rate": 0.0001999985665413352,
31
+ "loss": 4.4164,
32
+ "step": 100
33
+ },
34
+ {
35
+ "epoch": 0.2604166666666667,
36
+ "grad_norm": 5.78125,
37
+ "learning_rate": 0.00019999414859436728,
38
+ "loss": 4.1765,
39
+ "step": 150
40
+ },
41
+ {
42
+ "epoch": 0.3472222222222222,
43
+ "grad_norm": 11.0,
44
+ "learning_rate": 0.00019998674569395055,
45
+ "loss": 4.0896,
46
+ "step": 200
47
+ },
48
+ {
49
+ "epoch": 0.4340277777777778,
50
+ "grad_norm": 6.625,
51
+ "learning_rate": 0.000199976358061071,
52
+ "loss": 3.9586,
53
+ "step": 250
54
+ },
55
+ {
56
+ "epoch": 0.5208333333333334,
57
+ "grad_norm": 6.65625,
58
+ "learning_rate": 0.00019996298600581287,
59
+ "loss": 3.9273,
60
+ "step": 300
61
+ },
62
+ {
63
+ "epoch": 0.6076388888888888,
64
+ "grad_norm": 13.125,
65
+ "learning_rate": 0.0001999466299273491,
66
+ "loss": 3.8612,
67
+ "step": 350
68
+ },
69
+ {
70
+ "epoch": 0.6944444444444444,
71
+ "grad_norm": 7.0625,
72
+ "learning_rate": 0.00019992729031392958,
73
+ "loss": 3.8205,
74
+ "step": 400
75
+ },
76
+ {
77
+ "epoch": 0.78125,
78
+ "grad_norm": 8.75,
79
+ "learning_rate": 0.00019990496774286654,
80
+ "loss": 3.7956,
81
+ "step": 450
82
+ },
83
+ {
84
+ "epoch": 0.8680555555555556,
85
+ "grad_norm": 8.75,
86
+ "learning_rate": 0.00019987966288051735,
87
+ "loss": 3.7654,
88
+ "step": 500
89
+ },
90
+ {
91
+ "epoch": 0.9548611111111112,
92
+ "grad_norm": 14.0625,
93
+ "learning_rate": 0.00019985137648226457,
94
+ "loss": 3.6055,
95
+ "step": 550
96
+ },
97
+ {
98
+ "epoch": 1.0,
99
+ "eval_loss": 3.320380210876465,
100
+ "eval_runtime": 41.8114,
101
+ "eval_samples_per_second": 89.186,
102
+ "eval_steps_per_second": 5.597,
103
+ "step": 576
104
+ },
105
+ {
106
+ "epoch": 1.0416666666666667,
107
+ "grad_norm": 13.875,
108
+ "learning_rate": 0.00019982010939249346,
109
+ "loss": 3.4141,
110
+ "step": 600
111
+ },
112
+ {
113
+ "epoch": 1.1284722222222223,
114
+ "grad_norm": 15.125,
115
+ "learning_rate": 0.0001997858625445666,
116
+ "loss": 3.3461,
117
+ "step": 650
118
+ },
119
+ {
120
+ "epoch": 1.2152777777777777,
121
+ "grad_norm": 13.25,
122
+ "learning_rate": 0.0001997486369607964,
123
+ "loss": 3.2968,
124
+ "step": 700
125
+ },
126
+ {
127
+ "epoch": 1.3020833333333333,
128
+ "grad_norm": 11.25,
129
+ "learning_rate": 0.00019970843375241416,
130
+ "loss": 3.2924,
131
+ "step": 750
132
+ },
133
+ {
134
+ "epoch": 1.3888888888888888,
135
+ "grad_norm": 12.5,
136
+ "learning_rate": 0.00019966525411953717,
137
+ "loss": 3.2577,
138
+ "step": 800
139
+ },
140
+ {
141
+ "epoch": 1.4756944444444444,
142
+ "grad_norm": 13.0625,
143
+ "learning_rate": 0.00019961909935113284,
144
+ "loss": 3.2544,
145
+ "step": 850
146
+ },
147
+ {
148
+ "epoch": 1.5625,
149
+ "grad_norm": 14.125,
150
+ "learning_rate": 0.00019956997082498009,
151
+ "loss": 3.2245,
152
+ "step": 900
153
+ },
154
+ {
155
+ "epoch": 1.6493055555555556,
156
+ "grad_norm": 9.1875,
157
+ "learning_rate": 0.00019951787000762835,
158
+ "loss": 3.2121,
159
+ "step": 950
160
+ },
161
+ {
162
+ "epoch": 1.7361111111111112,
163
+ "grad_norm": 13.125,
164
+ "learning_rate": 0.00019946279845435382,
165
+ "loss": 3.1861,
166
+ "step": 1000
167
+ },
168
+ {
169
+ "epoch": 1.8229166666666665,
170
+ "grad_norm": 8.8125,
171
+ "learning_rate": 0.0001994047578091129,
172
+ "loss": 3.1813,
173
+ "step": 1050
174
+ },
175
+ {
176
+ "epoch": 1.9097222222222223,
177
+ "grad_norm": 10.5,
178
+ "learning_rate": 0.00019934374980449325,
179
+ "loss": 3.1483,
180
+ "step": 1100
181
+ },
182
+ {
183
+ "epoch": 1.9965277777777777,
184
+ "grad_norm": 11.875,
185
+ "learning_rate": 0.00019927977626166193,
186
+ "loss": 3.1491,
187
+ "step": 1150
188
+ },
189
+ {
190
+ "epoch": 2.0,
191
+ "eval_loss": 3.0681025981903076,
192
+ "eval_runtime": 41.9062,
193
+ "eval_samples_per_second": 88.984,
194
+ "eval_steps_per_second": 5.584,
195
+ "step": 1152
196
+ },
197
+ {
198
+ "epoch": 2.0833333333333335,
199
+ "grad_norm": 10.25,
200
+ "learning_rate": 0.00019921283909031114,
201
+ "loss": 3.1364,
202
+ "step": 1200
203
+ },
204
+ {
205
+ "epoch": 2.170138888888889,
206
+ "grad_norm": 10.375,
207
+ "learning_rate": 0.00019914294028860127,
208
+ "loss": 3.1123,
209
+ "step": 1250
210
+ },
211
+ {
212
+ "epoch": 2.2569444444444446,
213
+ "grad_norm": 10.3125,
214
+ "learning_rate": 0.00019907008194310102,
215
+ "loss": 3.1234,
216
+ "step": 1300
217
+ },
218
+ {
219
+ "epoch": 2.34375,
220
+ "grad_norm": 12.9375,
221
+ "learning_rate": 0.00019899426622872543,
222
+ "loss": 3.1215,
223
+ "step": 1350
224
+ },
225
+ {
226
+ "epoch": 2.4305555555555554,
227
+ "grad_norm": 12.5,
228
+ "learning_rate": 0.00019891549540867066,
229
+ "loss": 3.0999,
230
+ "step": 1400
231
+ },
232
+ {
233
+ "epoch": 2.517361111111111,
234
+ "grad_norm": 6.59375,
235
+ "learning_rate": 0.00019883377183434666,
236
+ "loss": 3.1192,
237
+ "step": 1450
238
+ },
239
+ {
240
+ "epoch": 2.6041666666666665,
241
+ "grad_norm": 7.6875,
242
+ "learning_rate": 0.00019874909794530675,
243
+ "loss": 3.0983,
244
+ "step": 1500
245
+ },
246
+ {
247
+ "epoch": 2.6909722222222223,
248
+ "grad_norm": 8.625,
249
+ "learning_rate": 0.0001986614762691751,
250
+ "loss": 3.0853,
251
+ "step": 1550
252
+ },
253
+ {
254
+ "epoch": 2.7777777777777777,
255
+ "grad_norm": 12.875,
256
+ "learning_rate": 0.00019857090942157092,
257
+ "loss": 3.0822,
258
+ "step": 1600
259
+ },
260
+ {
261
+ "epoch": 2.8645833333333335,
262
+ "grad_norm": 11.3125,
263
+ "learning_rate": 0.00019847740010603068,
264
+ "loss": 3.0779,
265
+ "step": 1650
266
+ },
267
+ {
268
+ "epoch": 2.951388888888889,
269
+ "grad_norm": 7.3125,
270
+ "learning_rate": 0.00019838095111392726,
271
+ "loss": 3.0747,
272
+ "step": 1700
273
+ },
274
+ {
275
+ "epoch": 3.0,
276
+ "eval_loss": 3.002568483352661,
277
+ "eval_runtime": 40.5832,
278
+ "eval_samples_per_second": 91.885,
279
+ "eval_steps_per_second": 5.766,
280
+ "step": 1728
281
+ },
282
+ {
283
+ "epoch": 3.0381944444444446,
284
+ "grad_norm": 11.625,
285
+ "learning_rate": 0.00019828156532438666,
286
+ "loss": 3.0638,
287
+ "step": 1750
288
+ },
289
+ {
290
+ "epoch": 3.125,
291
+ "grad_norm": 10.9375,
292
+ "learning_rate": 0.00019817924570420198,
293
+ "loss": 3.0585,
294
+ "step": 1800
295
+ },
296
+ {
297
+ "epoch": 3.2118055555555554,
298
+ "grad_norm": 7.0625,
299
+ "learning_rate": 0.00019807399530774502,
300
+ "loss": 3.0494,
301
+ "step": 1850
302
+ },
303
+ {
304
+ "epoch": 3.298611111111111,
305
+ "grad_norm": 9.125,
306
+ "learning_rate": 0.00019796581727687493,
307
+ "loss": 3.0628,
308
+ "step": 1900
309
+ },
310
+ {
311
+ "epoch": 3.3854166666666665,
312
+ "grad_norm": 11.875,
313
+ "learning_rate": 0.00019785471484084458,
314
+ "loss": 3.0529,
315
+ "step": 1950
316
+ },
317
+ {
318
+ "epoch": 3.4722222222222223,
319
+ "grad_norm": 14.9375,
320
+ "learning_rate": 0.00019774069131620398,
321
+ "loss": 3.0594,
322
+ "step": 2000
323
+ },
324
+ {
325
+ "epoch": 3.5590277777777777,
326
+ "grad_norm": 8.4375,
327
+ "learning_rate": 0.00019762375010670143,
328
+ "loss": 3.0478,
329
+ "step": 2050
330
+ },
331
+ {
332
+ "epoch": 3.6458333333333335,
333
+ "grad_norm": 9.8125,
334
+ "learning_rate": 0.0001975038947031819,
335
+ "loss": 3.0401,
336
+ "step": 2100
337
+ },
338
+ {
339
+ "epoch": 3.732638888888889,
340
+ "grad_norm": 11.0,
341
+ "learning_rate": 0.0001973811286834827,
342
+ "loss": 3.0339,
343
+ "step": 2150
344
+ },
345
+ {
346
+ "epoch": 3.8194444444444446,
347
+ "grad_norm": 9.0625,
348
+ "learning_rate": 0.00019725545571232686,
349
+ "loss": 3.0461,
350
+ "step": 2200
351
+ },
352
+ {
353
+ "epoch": 3.90625,
354
+ "grad_norm": 7.21875,
355
+ "learning_rate": 0.0001971268795412135,
356
+ "loss": 3.0156,
357
+ "step": 2250
358
+ },
359
+ {
360
+ "epoch": 3.9930555555555554,
361
+ "grad_norm": 9.75,
362
+ "learning_rate": 0.00019699540400830616,
363
+ "loss": 3.0261,
364
+ "step": 2300
365
+ },
366
+ {
367
+ "epoch": 4.0,
368
+ "eval_loss": 2.960036516189575,
369
+ "eval_runtime": 41.7286,
370
+ "eval_samples_per_second": 89.363,
371
+ "eval_steps_per_second": 5.608,
372
+ "step": 2304
373
+ },
374
+ {
375
+ "epoch": 4.079861111111111,
376
+ "grad_norm": 7.53125,
377
+ "learning_rate": 0.00019686103303831787,
378
+ "loss": 3.0194,
379
+ "step": 2350
380
+ },
381
+ {
382
+ "epoch": 4.166666666666667,
383
+ "grad_norm": 8.0,
384
+ "learning_rate": 0.0001967237706423943,
385
+ "loss": 2.9982,
386
+ "step": 2400
387
+ },
388
+ {
389
+ "epoch": 4.253472222222222,
390
+ "grad_norm": 10.0,
391
+ "learning_rate": 0.00019658362091799374,
392
+ "loss": 3.0147,
393
+ "step": 2450
394
+ },
395
+ {
396
+ "epoch": 4.340277777777778,
397
+ "grad_norm": 8.9375,
398
+ "learning_rate": 0.00019644058804876513,
399
+ "loss": 3.0187,
400
+ "step": 2500
401
+ },
402
+ {
403
+ "epoch": 4.427083333333333,
404
+ "grad_norm": 7.28125,
405
+ "learning_rate": 0.0001962946763044228,
406
+ "loss": 3.0009,
407
+ "step": 2550
408
+ },
409
+ {
410
+ "epoch": 4.513888888888889,
411
+ "grad_norm": 6.96875,
412
+ "learning_rate": 0.00019614589004061928,
413
+ "loss": 3.0264,
414
+ "step": 2600
415
+ },
416
+ {
417
+ "epoch": 4.600694444444445,
418
+ "grad_norm": 8.6875,
419
+ "learning_rate": 0.0001959942336988152,
420
+ "loss": 3.0037,
421
+ "step": 2650
422
+ },
423
+ {
424
+ "epoch": 4.6875,
425
+ "grad_norm": 8.25,
426
+ "learning_rate": 0.0001958397118061466,
427
+ "loss": 3.0003,
428
+ "step": 2700
429
+ },
430
+ {
431
+ "epoch": 4.774305555555555,
432
+ "grad_norm": 7.1875,
433
+ "learning_rate": 0.00019568232897529002,
434
+ "loss": 2.9937,
435
+ "step": 2750
436
+ },
437
+ {
438
+ "epoch": 4.861111111111111,
439
+ "grad_norm": 7.5,
440
+ "learning_rate": 0.00019552208990432457,
441
+ "loss": 2.9977,
442
+ "step": 2800
443
+ },
444
+ {
445
+ "epoch": 4.947916666666667,
446
+ "grad_norm": 11.8125,
447
+ "learning_rate": 0.0001953589993765918,
448
+ "loss": 2.992,
449
+ "step": 2850
450
+ },
451
+ {
452
+ "epoch": 5.0,
453
+ "eval_loss": 2.9334027767181396,
454
+ "eval_runtime": 42.3875,
455
+ "eval_samples_per_second": 87.974,
456
+ "eval_steps_per_second": 5.52,
457
+ "step": 2880
458
+ },
459
+ {
460
+ "epoch": 5.034722222222222,
461
+ "grad_norm": 8.8125,
462
+ "learning_rate": 0.000195193062260553,
463
+ "loss": 2.9851,
464
+ "step": 2900
465
+ },
466
+ {
467
+ "epoch": 5.121527777777778,
468
+ "grad_norm": 7.875,
469
+ "learning_rate": 0.00019502428350964355,
470
+ "loss": 2.9796,
471
+ "step": 2950
472
+ },
473
+ {
474
+ "epoch": 5.208333333333333,
475
+ "grad_norm": 6.53125,
476
+ "learning_rate": 0.00019485266816212548,
477
+ "loss": 2.977,
478
+ "step": 3000
479
+ },
480
+ {
481
+ "epoch": 5.295138888888889,
482
+ "grad_norm": 11.125,
483
+ "learning_rate": 0.00019467822134093684,
484
+ "loss": 2.9887,
485
+ "step": 3050
486
+ },
487
+ {
488
+ "epoch": 5.381944444444445,
489
+ "grad_norm": 8.0625,
490
+ "learning_rate": 0.00019450094825353864,
491
+ "loss": 2.982,
492
+ "step": 3100
493
+ },
494
+ {
495
+ "epoch": 5.46875,
496
+ "grad_norm": 8.75,
497
+ "learning_rate": 0.00019432085419175975,
498
+ "loss": 2.9896,
499
+ "step": 3150
500
+ },
501
+ {
502
+ "epoch": 5.555555555555555,
503
+ "grad_norm": 8.0625,
504
+ "learning_rate": 0.00019413794453163857,
505
+ "loss": 2.9854,
506
+ "step": 3200
507
+ },
508
+ {
509
+ "epoch": 5.642361111111111,
510
+ "grad_norm": 10.4375,
511
+ "learning_rate": 0.00019395222473326284,
512
+ "loss": 2.9749,
513
+ "step": 3250
514
+ },
515
+ {
516
+ "epoch": 5.729166666666667,
517
+ "grad_norm": 7.03125,
518
+ "learning_rate": 0.00019376370034060653,
519
+ "loss": 2.9705,
520
+ "step": 3300
521
+ },
522
+ {
523
+ "epoch": 5.815972222222222,
524
+ "grad_norm": 9.8125,
525
+ "learning_rate": 0.00019357237698136427,
526
+ "loss": 2.9855,
527
+ "step": 3350
528
+ },
529
+ {
530
+ "epoch": 5.902777777777778,
531
+ "grad_norm": 6.78125,
532
+ "learning_rate": 0.00019337826036678338,
533
+ "loss": 2.9596,
534
+ "step": 3400
535
+ },
536
+ {
537
+ "epoch": 5.989583333333333,
538
+ "grad_norm": 8.6875,
539
+ "learning_rate": 0.00019318135629149363,
540
+ "loss": 2.9692,
541
+ "step": 3450
542
+ },
543
+ {
544
+ "epoch": 6.0,
545
+ "eval_loss": 2.9161436557769775,
546
+ "eval_runtime": 41.8777,
547
+ "eval_samples_per_second": 89.045,
548
+ "eval_steps_per_second": 5.588,
549
+ "step": 3456
550
+ },
551
+ {
552
+ "epoch": 6.076388888888889,
553
+ "grad_norm": 8.5625,
554
+ "learning_rate": 0.0001929816706333339,
555
+ "loss": 2.9666,
556
+ "step": 3500
557
+ },
558
+ {
559
+ "epoch": 6.163194444444445,
560
+ "grad_norm": 11.625,
561
+ "learning_rate": 0.00019277920935317688,
562
+ "loss": 2.9451,
563
+ "step": 3550
564
+ },
565
+ {
566
+ "epoch": 6.25,
567
+ "grad_norm": 7.625,
568
+ "learning_rate": 0.00019257397849475124,
569
+ "loss": 2.9624,
570
+ "step": 3600
571
+ },
572
+ {
573
+ "epoch": 6.336805555555555,
574
+ "grad_norm": 7.34375,
575
+ "learning_rate": 0.00019236598418446098,
576
+ "loss": 2.9722,
577
+ "step": 3650
578
+ },
579
+ {
580
+ "epoch": 6.423611111111111,
581
+ "grad_norm": 7.5,
582
+ "learning_rate": 0.00019215523263120283,
583
+ "loss": 2.9552,
584
+ "step": 3700
585
+ },
586
+ {
587
+ "epoch": 6.510416666666667,
588
+ "grad_norm": 10.625,
589
+ "learning_rate": 0.0001919417301261806,
590
+ "loss": 2.9844,
591
+ "step": 3750
592
+ },
593
+ {
594
+ "epoch": 6.597222222222222,
595
+ "grad_norm": 6.25,
596
+ "learning_rate": 0.00019172548304271768,
597
+ "loss": 2.9576,
598
+ "step": 3800
599
+ },
600
+ {
601
+ "epoch": 6.684027777777778,
602
+ "grad_norm": 8.25,
603
+ "learning_rate": 0.00019150649783606646,
604
+ "loss": 2.9598,
605
+ "step": 3850
606
+ },
607
+ {
608
+ "epoch": 6.770833333333333,
609
+ "grad_norm": 6.25,
610
+ "learning_rate": 0.00019128478104321603,
611
+ "loss": 2.9488,
612
+ "step": 3900
613
+ },
614
+ {
615
+ "epoch": 6.857638888888889,
616
+ "grad_norm": 8.25,
617
+ "learning_rate": 0.00019106033928269667,
618
+ "loss": 2.9591,
619
+ "step": 3950
620
+ },
621
+ {
622
+ "epoch": 6.944444444444445,
623
+ "grad_norm": 5.8125,
624
+ "learning_rate": 0.00019083317925438248,
625
+ "loss": 2.9501,
626
+ "step": 4000
627
+ },
628
+ {
629
+ "epoch": 7.0,
630
+ "eval_loss": 2.90425968170166,
631
+ "eval_runtime": 41.3276,
632
+ "eval_samples_per_second": 90.23,
633
+ "eval_steps_per_second": 5.662,
634
+ "step": 4032
635
+ },
636
+ {
637
+ "epoch": 7.03125,
638
+ "grad_norm": 6.40625,
639
+ "learning_rate": 0.00019060330773929137,
640
+ "loss": 2.9478,
641
+ "step": 4050
642
+ },
643
+ {
644
+ "epoch": 7.118055555555555,
645
+ "grad_norm": 8.75,
646
+ "learning_rate": 0.00019037073159938256,
647
+ "loss": 2.9421,
648
+ "step": 4100
649
+ },
650
+ {
651
+ "epoch": 7.204861111111111,
652
+ "grad_norm": 6.1875,
653
+ "learning_rate": 0.00019013545777735183,
654
+ "loss": 2.9394,
655
+ "step": 4150
656
+ },
657
+ {
658
+ "epoch": 7.291666666666667,
659
+ "grad_norm": 6.71875,
660
+ "learning_rate": 0.00018989749329642418,
661
+ "loss": 2.9519,
662
+ "step": 4200
663
+ },
664
+ {
665
+ "epoch": 7.378472222222222,
666
+ "grad_norm": 6.25,
667
+ "learning_rate": 0.00018965684526014425,
668
+ "loss": 2.9475,
669
+ "step": 4250
670
+ },
671
+ {
672
+ "epoch": 7.465277777777778,
673
+ "grad_norm": 4.46875,
674
+ "learning_rate": 0.00018941352085216425,
675
+ "loss": 2.9507,
676
+ "step": 4300
677
+ },
678
+ {
679
+ "epoch": 7.552083333333333,
680
+ "grad_norm": 7.5625,
681
+ "learning_rate": 0.0001891675273360295,
682
+ "loss": 2.956,
683
+ "step": 4350
684
+ },
685
+ {
686
+ "epoch": 7.638888888888889,
687
+ "grad_norm": 8.1875,
688
+ "learning_rate": 0.00018891887205496163,
689
+ "loss": 2.9422,
690
+ "step": 4400
691
+ },
692
+ {
693
+ "epoch": 7.725694444444445,
694
+ "grad_norm": 6.625,
695
+ "learning_rate": 0.00018866756243163938,
696
+ "loss": 2.9379,
697
+ "step": 4450
698
+ },
699
+ {
700
+ "epoch": 7.8125,
701
+ "grad_norm": 7.46875,
702
+ "learning_rate": 0.00018841360596797695,
703
+ "loss": 2.9477,
704
+ "step": 4500
705
+ },
706
+ {
707
+ "epoch": 7.899305555555555,
708
+ "grad_norm": 9.3125,
709
+ "learning_rate": 0.0001881570102449002,
710
+ "loss": 2.9293,
711
+ "step": 4550
712
+ },
713
+ {
714
+ "epoch": 7.986111111111111,
715
+ "grad_norm": 8.375,
716
+ "learning_rate": 0.0001878977829221201,
717
+ "loss": 2.9379,
718
+ "step": 4600
719
+ },
720
+ {
721
+ "epoch": 8.0,
722
+ "eval_loss": 2.894627571105957,
723
+ "eval_runtime": 42.0326,
724
+ "eval_samples_per_second": 88.717,
725
+ "eval_steps_per_second": 5.567,
726
+ "step": 4608
727
+ },
728
+ {
729
+ "epoch": 8.072916666666666,
730
+ "grad_norm": 7.625,
731
+ "learning_rate": 0.00018763593173790454,
732
+ "loss": 2.9327,
733
+ "step": 4650
734
+ },
735
+ {
736
+ "epoch": 8.159722222222221,
737
+ "grad_norm": 6.25,
738
+ "learning_rate": 0.00018737146450884668,
739
+ "loss": 2.917,
740
+ "step": 4700
741
+ },
742
+ {
743
+ "epoch": 8.246527777777779,
744
+ "grad_norm": 5.28125,
745
+ "learning_rate": 0.00018710438912963225,
746
+ "loss": 2.9335,
747
+ "step": 4750
748
+ },
749
+ {
750
+ "epoch": 8.333333333333334,
751
+ "grad_norm": 5.90625,
752
+ "learning_rate": 0.00018683471357280347,
753
+ "loss": 2.9416,
754
+ "step": 4800
755
+ },
756
+ {
757
+ "epoch": 8.42013888888889,
758
+ "grad_norm": 6.84375,
759
+ "learning_rate": 0.00018656244588852124,
760
+ "loss": 2.9256,
761
+ "step": 4850
762
+ },
763
+ {
764
+ "epoch": 8.506944444444445,
765
+ "grad_norm": 5.5625,
766
+ "learning_rate": 0.00018628759420432473,
767
+ "loss": 2.9525,
768
+ "step": 4900
769
+ },
770
+ {
771
+ "epoch": 8.59375,
772
+ "grad_norm": 6.0625,
773
+ "learning_rate": 0.00018601016672488888,
774
+ "loss": 2.9268,
775
+ "step": 4950
776
+ },
777
+ {
778
+ "epoch": 8.680555555555555,
779
+ "grad_norm": 6.90625,
780
+ "learning_rate": 0.00018573017173177938,
781
+ "loss": 2.9347,
782
+ "step": 5000
783
+ },
784
+ {
785
+ "epoch": 8.76736111111111,
786
+ "grad_norm": 5.78125,
787
+ "learning_rate": 0.0001854476175832055,
788
+ "loss": 2.9267,
789
+ "step": 5050
790
+ },
791
+ {
792
+ "epoch": 8.854166666666666,
793
+ "grad_norm": 6.375,
794
+ "learning_rate": 0.00018516251271377064,
795
+ "loss": 2.9246,
796
+ "step": 5100
797
+ },
798
+ {
799
+ "epoch": 8.940972222222221,
800
+ "grad_norm": 6.0,
801
+ "learning_rate": 0.00018487486563422036,
802
+ "loss": 2.9221,
803
+ "step": 5150
804
+ },
805
+ {
806
+ "epoch": 9.0,
807
+ "eval_loss": 2.8883821964263916,
808
+ "eval_runtime": 41.385,
809
+ "eval_samples_per_second": 90.105,
810
+ "eval_steps_per_second": 5.654,
811
+ "step": 5184
812
+ },
813
+ {
814
+ "epoch": 9.027777777777779,
815
+ "grad_norm": 7.875,
816
+ "learning_rate": 0.00018458468493118857,
817
+ "loss": 2.9219,
818
+ "step": 5200
819
+ },
820
+ {
821
+ "epoch": 9.114583333333334,
822
+ "grad_norm": 7.90625,
823
+ "learning_rate": 0.000184291979266941,
824
+ "loss": 2.9209,
825
+ "step": 5250
826
+ },
827
+ {
828
+ "epoch": 9.20138888888889,
829
+ "grad_norm": 6.4375,
830
+ "learning_rate": 0.00018399675737911677,
831
+ "loss": 2.9127,
832
+ "step": 5300
833
+ },
834
+ {
835
+ "epoch": 9.288194444444445,
836
+ "grad_norm": 5.9375,
837
+ "learning_rate": 0.00018369902808046748,
838
+ "loss": 2.9262,
839
+ "step": 5350
840
+ },
841
+ {
842
+ "epoch": 9.375,
843
+ "grad_norm": 6.5625,
844
+ "learning_rate": 0.0001833988002585941,
845
+ "loss": 2.9258,
846
+ "step": 5400
847
+ },
848
+ {
849
+ "epoch": 9.461805555555555,
850
+ "grad_norm": 5.8125,
851
+ "learning_rate": 0.00018309608287568182,
852
+ "loss": 2.9275,
853
+ "step": 5450
854
+ },
855
+ {
856
+ "epoch": 9.54861111111111,
857
+ "grad_norm": 6.25,
858
+ "learning_rate": 0.00018279088496823235,
859
+ "loss": 2.9312,
860
+ "step": 5500
861
+ },
862
+ {
863
+ "epoch": 9.635416666666666,
864
+ "grad_norm": 6.09375,
865
+ "learning_rate": 0.00018248321564679425,
866
+ "loss": 2.9205,
867
+ "step": 5550
868
+ },
869
+ {
870
+ "epoch": 9.722222222222221,
871
+ "grad_norm": 8.0625,
872
+ "learning_rate": 0.0001821730840956909,
873
+ "loss": 2.9203,
874
+ "step": 5600
875
+ },
876
+ {
877
+ "epoch": 9.809027777777779,
878
+ "grad_norm": 4.6875,
879
+ "learning_rate": 0.00018186049957274656,
880
+ "loss": 2.9264,
881
+ "step": 5650
882
+ },
883
+ {
884
+ "epoch": 9.895833333333334,
885
+ "grad_norm": 5.0,
886
+ "learning_rate": 0.0001815454714090096,
887
+ "loss": 2.9109,
888
+ "step": 5700
889
+ },
890
+ {
891
+ "epoch": 9.98263888888889,
892
+ "grad_norm": 5.875,
893
+ "learning_rate": 0.0001812280090084744,
894
+ "loss": 2.9139,
895
+ "step": 5750
896
+ },
897
+ {
898
+ "epoch": 10.0,
899
+ "eval_loss": 2.8820853233337402,
900
+ "eval_runtime": 42.0383,
901
+ "eval_samples_per_second": 88.705,
902
+ "eval_steps_per_second": 5.566,
903
+ "step": 5760
904
+ },
905
+ {
906
+ "epoch": 10.069444444444445,
907
+ "grad_norm": 5.90625,
908
+ "learning_rate": 0.00018090812184780032,
909
+ "loss": 2.9105,
910
+ "step": 5800
911
+ },
912
+ {
913
+ "epoch": 10.15625,
914
+ "grad_norm": 5.59375,
915
+ "learning_rate": 0.000180585819476029,
916
+ "loss": 2.9039,
917
+ "step": 5850
918
+ },
919
+ {
920
+ "epoch": 10.243055555555555,
921
+ "grad_norm": 5.84375,
922
+ "learning_rate": 0.0001802611115142991,
923
+ "loss": 2.9122,
924
+ "step": 5900
925
+ },
926
+ {
927
+ "epoch": 10.32986111111111,
928
+ "grad_norm": 6.75,
929
+ "learning_rate": 0.00017993400765555932,
930
+ "loss": 2.9233,
931
+ "step": 5950
932
+ },
933
+ {
934
+ "epoch": 10.416666666666666,
935
+ "grad_norm": 5.71875,
936
+ "learning_rate": 0.00017960451766427897,
937
+ "loss": 2.9075,
938
+ "step": 6000
939
+ },
940
+ {
941
+ "epoch": 10.503472222222221,
942
+ "grad_norm": 6.09375,
943
+ "learning_rate": 0.00017927265137615637,
944
+ "loss": 2.937,
945
+ "step": 6050
946
+ },
947
+ {
948
+ "epoch": 10.590277777777779,
949
+ "grad_norm": 4.90625,
950
+ "learning_rate": 0.00017893841869782547,
951
+ "loss": 2.9075,
952
+ "step": 6100
953
+ },
954
+ {
955
+ "epoch": 10.677083333333334,
956
+ "grad_norm": 5.5625,
957
+ "learning_rate": 0.0001786018296065599,
958
+ "loss": 2.9184,
959
+ "step": 6150
960
+ },
961
+ {
962
+ "epoch": 10.76388888888889,
963
+ "grad_norm": 5.71875,
964
+ "learning_rate": 0.0001782628941499753,
965
+ "loss": 2.9093,
966
+ "step": 6200
967
+ },
968
+ {
969
+ "epoch": 10.850694444444445,
970
+ "grad_norm": 6.9375,
971
+ "learning_rate": 0.00017792162244572928,
972
+ "loss": 2.911,
973
+ "step": 6250
974
+ },
975
+ {
976
+ "epoch": 10.9375,
977
+ "grad_norm": 8.125,
978
+ "learning_rate": 0.00017757802468121946,
979
+ "loss": 2.9023,
980
+ "step": 6300
981
+ },
982
+ {
983
+ "epoch": 11.0,
984
+ "eval_loss": 2.8765242099761963,
985
+ "eval_runtime": 40.8481,
986
+ "eval_samples_per_second": 91.289,
987
+ "eval_steps_per_second": 5.729,
988
+ "step": 6336
989
+ },
990
+ {
991
+ "epoch": 11.024305555555555,
992
+ "grad_norm": 4.3125,
993
+ "learning_rate": 0.00017723211111327934,
994
+ "loss": 2.9075,
995
+ "step": 6350
996
+ },
997
+ {
998
+ "epoch": 11.11111111111111,
999
+ "grad_norm": 4.6875,
1000
+ "learning_rate": 0.0001768838920678721,
1001
+ "loss": 2.9027,
1002
+ "step": 6400
1003
+ },
1004
+ {
1005
+ "epoch": 11.197916666666666,
1006
+ "grad_norm": 8.375,
1007
+ "learning_rate": 0.00017653337793978237,
1008
+ "loss": 2.8971,
1009
+ "step": 6450
1010
+ },
1011
+ {
1012
+ "epoch": 11.284722222222221,
1013
+ "grad_norm": 6.34375,
1014
+ "learning_rate": 0.00017618057919230597,
1015
+ "loss": 2.9095,
1016
+ "step": 6500
1017
+ },
1018
+ {
1019
+ "epoch": 11.371527777777779,
1020
+ "grad_norm": 10.125,
1021
+ "learning_rate": 0.00017582550635693753,
1022
+ "loss": 2.9108,
1023
+ "step": 6550
1024
+ },
1025
+ {
1026
+ "epoch": 11.458333333333334,
1027
+ "grad_norm": 9.375,
1028
+ "learning_rate": 0.0001754681700330561,
1029
+ "loss": 2.9115,
1030
+ "step": 6600
1031
+ },
1032
+ {
1033
+ "epoch": 11.54513888888889,
1034
+ "grad_norm": 5.96875,
1035
+ "learning_rate": 0.00017510858088760876,
1036
+ "loss": 2.9137,
1037
+ "step": 6650
1038
+ },
1039
+ {
1040
+ "epoch": 11.631944444444445,
1041
+ "grad_norm": 6.9375,
1042
+ "learning_rate": 0.00017474674965479222,
1043
+ "loss": 2.91,
1044
+ "step": 6700
1045
+ },
1046
+ {
1047
+ "epoch": 11.71875,
1048
+ "grad_norm": 9.8125,
1049
+ "learning_rate": 0.00017438268713573237,
1050
+ "loss": 2.9037,
1051
+ "step": 6750
1052
+ },
1053
+ {
1054
+ "epoch": 11.805555555555555,
1055
+ "grad_norm": 4.75,
1056
+ "learning_rate": 0.00017401640419816182,
1057
+ "loss": 2.9103,
1058
+ "step": 6800
1059
+ },
1060
+ {
1061
+ "epoch": 11.89236111111111,
1062
+ "grad_norm": 6.96875,
1063
+ "learning_rate": 0.00017364791177609554,
1064
+ "loss": 2.895,
1065
+ "step": 6850
1066
+ },
1067
+ {
1068
+ "epoch": 11.979166666666666,
1069
+ "grad_norm": 6.0625,
1070
+ "learning_rate": 0.00017327722086950446,
1071
+ "loss": 2.8989,
1072
+ "step": 6900
1073
+ },
1074
+ {
1075
+ "epoch": 12.0,
1076
+ "eval_loss": 2.872136116027832,
1077
+ "eval_runtime": 41.6305,
1078
+ "eval_samples_per_second": 89.574,
1079
+ "eval_steps_per_second": 5.621,
1080
+ "step": 6912
1081
+ },
1082
+ {
1083
+ "epoch": 12.065972222222221,
1084
+ "grad_norm": 7.75,
1085
+ "learning_rate": 0.0001729043425439871,
1086
+ "loss": 2.8952,
1087
+ "step": 6950
1088
+ },
1089
+ {
1090
+ "epoch": 12.152777777777779,
1091
+ "grad_norm": 5.84375,
1092
+ "learning_rate": 0.00017252928793043916,
1093
+ "loss": 2.8915,
1094
+ "step": 7000
1095
+ },
1096
+ {
1097
+ "epoch": 12.239583333333334,
1098
+ "grad_norm": 6.5625,
1099
+ "learning_rate": 0.00017215206822472143,
1100
+ "loss": 2.8955,
1101
+ "step": 7050
1102
+ },
1103
+ {
1104
+ "epoch": 12.32638888888889,
1105
+ "grad_norm": 5.875,
1106
+ "learning_rate": 0.00017177269468732535,
1107
+ "loss": 2.9131,
1108
+ "step": 7100
1109
+ },
1110
+ {
1111
+ "epoch": 12.413194444444445,
1112
+ "grad_norm": 6.65625,
1113
+ "learning_rate": 0.00017139117864303714,
1114
+ "loss": 2.8935,
1115
+ "step": 7150
1116
+ },
1117
+ {
1118
+ "epoch": 12.5,
1119
+ "grad_norm": 6.96875,
1120
+ "learning_rate": 0.0001710075314805995,
1121
+ "loss": 2.9223,
1122
+ "step": 7200
1123
+ },
1124
+ {
1125
+ "epoch": 12.586805555555555,
1126
+ "grad_norm": 5.71875,
1127
+ "learning_rate": 0.00017062176465237175,
1128
+ "loss": 2.8979,
1129
+ "step": 7250
1130
+ },
1131
+ {
1132
+ "epoch": 12.67361111111111,
1133
+ "grad_norm": 7.28125,
1134
+ "learning_rate": 0.00017023388967398796,
1135
+ "loss": 2.9076,
1136
+ "step": 7300
1137
+ },
1138
+ {
1139
+ "epoch": 12.760416666666666,
1140
+ "grad_norm": 6.0625,
1141
+ "learning_rate": 0.00016984391812401316,
1142
+ "loss": 2.8939,
1143
+ "step": 7350
1144
+ },
1145
+ {
1146
+ "epoch": 12.847222222222221,
1147
+ "grad_norm": 5.03125,
1148
+ "learning_rate": 0.00016945186164359782,
1149
+ "loss": 2.9007,
1150
+ "step": 7400
1151
+ },
1152
+ {
1153
+ "epoch": 12.934027777777779,
1154
+ "grad_norm": 5.46875,
1155
+ "learning_rate": 0.00016905773193613013,
1156
+ "loss": 2.891,
1157
+ "step": 7450
1158
+ },
1159
+ {
1160
+ "epoch": 13.0,
1161
+ "eval_loss": 2.869907855987549,
1162
+ "eval_runtime": 41.6939,
1163
+ "eval_samples_per_second": 89.437,
1164
+ "eval_steps_per_second": 5.612,
1165
+ "step": 7488
1166
+ },
1167
+ {
1168
+ "epoch": 13.020833333333334,
1169
+ "grad_norm": 5.375,
1170
+ "learning_rate": 0.00016866154076688683,
1171
+ "loss": 2.8958,
1172
+ "step": 7500
1173
+ },
1174
+ {
1175
+ "epoch": 13.10763888888889,
1176
+ "grad_norm": 5.03125,
1177
+ "learning_rate": 0.00016826329996268196,
1178
+ "loss": 2.8938,
1179
+ "step": 7550
1180
+ },
1181
+ {
1182
+ "epoch": 13.194444444444445,
1183
+ "grad_norm": 6.0625,
1184
+ "learning_rate": 0.00016786302141151368,
1185
+ "loss": 2.8862,
1186
+ "step": 7600
1187
+ },
1188
+ {
1189
+ "epoch": 13.28125,
1190
+ "grad_norm": 5.21875,
1191
+ "learning_rate": 0.00016746071706220966,
1192
+ "loss": 2.8969,
1193
+ "step": 7650
1194
+ },
1195
+ {
1196
+ "epoch": 13.368055555555555,
1197
+ "grad_norm": 6.8125,
1198
+ "learning_rate": 0.00016705639892407014,
1199
+ "loss": 2.9042,
1200
+ "step": 7700
1201
+ },
1202
+ {
1203
+ "epoch": 13.45486111111111,
1204
+ "grad_norm": 5.28125,
1205
+ "learning_rate": 0.00016665007906650948,
1206
+ "loss": 2.8953,
1207
+ "step": 7750
1208
+ },
1209
+ {
1210
+ "epoch": 13.541666666666666,
1211
+ "grad_norm": 8.1875,
1212
+ "learning_rate": 0.00016624176961869616,
1213
+ "loss": 2.908,
1214
+ "step": 7800
1215
+ },
1216
+ {
1217
+ "epoch": 13.628472222222221,
1218
+ "grad_norm": 5.34375,
1219
+ "learning_rate": 0.0001658314827691902,
1220
+ "loss": 2.8964,
1221
+ "step": 7850
1222
+ },
1223
+ {
1224
+ "epoch": 13.715277777777779,
1225
+ "grad_norm": 5.0,
1226
+ "learning_rate": 0.00016541923076557978,
1227
+ "loss": 2.8924,
1228
+ "step": 7900
1229
+ },
1230
+ {
1231
+ "epoch": 13.802083333333334,
1232
+ "grad_norm": 5.28125,
1233
+ "learning_rate": 0.0001650050259141154,
1234
+ "loss": 2.9024,
1235
+ "step": 7950
1236
+ },
1237
+ {
1238
+ "epoch": 13.88888888888889,
1239
+ "grad_norm": 4.71875,
1240
+ "learning_rate": 0.00016458888057934248,
1241
+ "loss": 2.884,
1242
+ "step": 8000
1243
+ },
1244
+ {
1245
+ "epoch": 13.975694444444445,
1246
+ "grad_norm": 11.4375,
1247
+ "learning_rate": 0.0001641708071837325,
1248
+ "loss": 2.8926,
1249
+ "step": 8050
1250
+ },
1251
+ {
1252
+ "epoch": 14.0,
1253
+ "eval_loss": 2.8657476902008057,
1254
+ "eval_runtime": 41.9302,
1255
+ "eval_samples_per_second": 88.934,
1256
+ "eval_steps_per_second": 5.581,
1257
+ "step": 8064
1258
+ },
1259
+ {
1260
+ "epoch": 14.0625,
1261
+ "grad_norm": 6.40625,
1262
+ "learning_rate": 0.00016375081820731193,
1263
+ "loss": 2.8867,
1264
+ "step": 8100
1265
+ },
1266
+ {
1267
+ "epoch": 14.149305555555555,
1268
+ "grad_norm": 4.625,
1269
+ "learning_rate": 0.00016332892618728986,
1270
+ "loss": 2.8829,
1271
+ "step": 8150
1272
+ },
1273
+ {
1274
+ "epoch": 14.23611111111111,
1275
+ "grad_norm": 4.1875,
1276
+ "learning_rate": 0.00016290514371768356,
1277
+ "loss": 2.8852,
1278
+ "step": 8200
1279
+ },
1280
+ {
1281
+ "epoch": 14.322916666666666,
1282
+ "grad_norm": 4.3125,
1283
+ "learning_rate": 0.0001624794834489427,
1284
+ "loss": 2.9058,
1285
+ "step": 8250
1286
+ },
1287
+ {
1288
+ "epoch": 14.409722222222221,
1289
+ "grad_norm": 4.53125,
1290
+ "learning_rate": 0.00016205195808757173,
1291
+ "loss": 2.8848,
1292
+ "step": 8300
1293
+ },
1294
+ {
1295
+ "epoch": 14.496527777777779,
1296
+ "grad_norm": 6.21875,
1297
+ "learning_rate": 0.00016162258039575033,
1298
+ "loss": 2.9088,
1299
+ "step": 8350
1300
+ },
1301
+ {
1302
+ "epoch": 14.583333333333334,
1303
+ "grad_norm": 7.34375,
1304
+ "learning_rate": 0.0001611913631909528,
1305
+ "loss": 2.8913,
1306
+ "step": 8400
1307
+ },
1308
+ {
1309
+ "epoch": 14.67013888888889,
1310
+ "grad_norm": 6.0,
1311
+ "learning_rate": 0.00016075831934556518,
1312
+ "loss": 2.9013,
1313
+ "step": 8450
1314
+ },
1315
+ {
1316
+ "epoch": 14.756944444444445,
1317
+ "grad_norm": 8.9375,
1318
+ "learning_rate": 0.00016032346178650105,
1319
+ "loss": 2.8843,
1320
+ "step": 8500
1321
+ },
1322
+ {
1323
+ "epoch": 14.84375,
1324
+ "grad_norm": 4.9375,
1325
+ "learning_rate": 0.0001598868034948157,
1326
+ "loss": 2.8901,
1327
+ "step": 8550
1328
+ },
1329
+ {
1330
+ "epoch": 14.930555555555555,
1331
+ "grad_norm": 5.40625,
1332
+ "learning_rate": 0.00015944835750531858,
1333
+ "loss": 2.8824,
1334
+ "step": 8600
1335
+ },
1336
+ {
1337
+ "epoch": 15.0,
1338
+ "eval_loss": 2.8647055625915527,
1339
+ "eval_runtime": 41.7282,
1340
+ "eval_samples_per_second": 89.364,
1341
+ "eval_steps_per_second": 5.608,
1342
+ "step": 8640
1343
+ },
1344
+ {
1345
+ "epoch": 15.01736111111111,
1346
+ "grad_norm": 8.125,
1347
+ "learning_rate": 0.0001590081369061842,
1348
+ "loss": 2.8874,
1349
+ "step": 8650
1350
+ },
1351
+ {
1352
+ "epoch": 15.104166666666666,
1353
+ "grad_norm": 6.375,
1354
+ "learning_rate": 0.00015856615483856153,
1355
+ "loss": 2.8822,
1356
+ "step": 8700
1357
+ },
1358
+ {
1359
+ "epoch": 15.190972222222221,
1360
+ "grad_norm": 6.21875,
1361
+ "learning_rate": 0.00015812242449618147,
1362
+ "loss": 2.8752,
1363
+ "step": 8750
1364
+ },
1365
+ {
1366
+ "epoch": 15.277777777777779,
1367
+ "grad_norm": 7.15625,
1368
+ "learning_rate": 0.0001576769591249633,
1369
+ "loss": 2.8873,
1370
+ "step": 8800
1371
+ },
1372
+ {
1373
+ "epoch": 15.364583333333334,
1374
+ "grad_norm": 5.0625,
1375
+ "learning_rate": 0.0001572297720226191,
1376
+ "loss": 2.8993,
1377
+ "step": 8850
1378
+ },
1379
+ {
1380
+ "epoch": 15.45138888888889,
1381
+ "grad_norm": 5.8125,
1382
+ "learning_rate": 0.00015678087653825675,
1383
+ "loss": 2.8854,
1384
+ "step": 8900
1385
+ },
1386
+ {
1387
+ "epoch": 15.538194444444445,
1388
+ "grad_norm": 4.03125,
1389
+ "learning_rate": 0.0001563302860719816,
1390
+ "loss": 2.8994,
1391
+ "step": 8950
1392
+ },
1393
+ {
1394
+ "epoch": 15.625,
1395
+ "grad_norm": 6.59375,
1396
+ "learning_rate": 0.00015587801407449648,
1397
+ "loss": 2.8893,
1398
+ "step": 9000
1399
+ },
1400
+ {
1401
+ "epoch": 15.711805555555555,
1402
+ "grad_norm": 6.25,
1403
+ "learning_rate": 0.0001554240740466998,
1404
+ "loss": 2.8871,
1405
+ "step": 9050
1406
+ },
1407
+ {
1408
+ "epoch": 15.79861111111111,
1409
+ "grad_norm": 10.0625,
1410
+ "learning_rate": 0.00015496847953928313,
1411
+ "loss": 2.8935,
1412
+ "step": 9100
1413
+ },
1414
+ {
1415
+ "epoch": 15.885416666666666,
1416
+ "grad_norm": 5.9375,
1417
+ "learning_rate": 0.00015451124415232615,
1418
+ "loss": 2.8775,
1419
+ "step": 9150
1420
+ },
1421
+ {
1422
+ "epoch": 15.972222222222221,
1423
+ "grad_norm": 7.65625,
1424
+ "learning_rate": 0.00015405238153489096,
1425
+ "loss": 2.8831,
1426
+ "step": 9200
1427
+ },
1428
+ {
1429
+ "epoch": 16.0,
1430
+ "eval_loss": 2.8630547523498535,
1431
+ "eval_runtime": 40.8978,
1432
+ "eval_samples_per_second": 91.179,
1433
+ "eval_steps_per_second": 5.722,
1434
+ "step": 9216
1435
+ },
1436
+ {
1437
+ "epoch": 16.05902777777778,
1438
+ "grad_norm": 5.78125,
1439
+ "learning_rate": 0.00015359190538461462,
1440
+ "loss": 2.88,
1441
+ "step": 9250
1442
+ },
1443
+ {
1444
+ "epoch": 16.145833333333332,
1445
+ "grad_norm": 5.46875,
1446
+ "learning_rate": 0.00015312982944730018,
1447
+ "loss": 2.8777,
1448
+ "step": 9300
1449
+ },
1450
+ {
1451
+ "epoch": 16.23263888888889,
1452
+ "grad_norm": 6.71875,
1453
+ "learning_rate": 0.00015266616751650642,
1454
+ "loss": 2.8785,
1455
+ "step": 9350
1456
+ },
1457
+ {
1458
+ "epoch": 16.319444444444443,
1459
+ "grad_norm": 4.46875,
1460
+ "learning_rate": 0.00015220093343313592,
1461
+ "loss": 2.8968,
1462
+ "step": 9400
1463
+ },
1464
+ {
1465
+ "epoch": 16.40625,
1466
+ "grad_norm": 5.125,
1467
+ "learning_rate": 0.00015173414108502224,
1468
+ "loss": 2.877,
1469
+ "step": 9450
1470
+ },
1471
+ {
1472
+ "epoch": 16.493055555555557,
1473
+ "grad_norm": 5.03125,
1474
+ "learning_rate": 0.00015126580440651496,
1475
+ "loss": 2.9016,
1476
+ "step": 9500
1477
+ },
1478
+ {
1479
+ "epoch": 16.57986111111111,
1480
+ "grad_norm": 6.71875,
1481
+ "learning_rate": 0.00015079593737806399,
1482
+ "loss": 2.8841,
1483
+ "step": 9550
1484
+ },
1485
+ {
1486
+ "epoch": 16.666666666666668,
1487
+ "grad_norm": 5.21875,
1488
+ "learning_rate": 0.00015032455402580217,
1489
+ "loss": 2.8937,
1490
+ "step": 9600
1491
+ },
1492
+ {
1493
+ "epoch": 16.75347222222222,
1494
+ "grad_norm": 5.59375,
1495
+ "learning_rate": 0.00014985166842112644,
1496
+ "loss": 2.8789,
1497
+ "step": 9650
1498
+ },
1499
+ {
1500
+ "epoch": 16.84027777777778,
1501
+ "grad_norm": 5.5625,
1502
+ "learning_rate": 0.00014937729468027797,
1503
+ "loss": 2.8883,
1504
+ "step": 9700
1505
+ },
1506
+ {
1507
+ "epoch": 16.927083333333332,
1508
+ "grad_norm": 5.15625,
1509
+ "learning_rate": 0.00014890144696392074,
1510
+ "loss": 2.8751,
1511
+ "step": 9750
1512
+ },
1513
+ {
1514
+ "epoch": 17.0,
1515
+ "eval_loss": 2.862104892730713,
1516
+ "eval_runtime": 41.9728,
1517
+ "eval_samples_per_second": 88.843,
1518
+ "eval_steps_per_second": 5.575,
1519
+ "step": 9792
1520
+ },
1521
+ {
1522
+ "epoch": 17.01388888888889,
1523
+ "grad_norm": 8.0625,
1524
+ "learning_rate": 0.00014842413947671872,
1525
+ "loss": 2.8821,
1526
+ "step": 9800
1527
+ },
1528
+ {
1529
+ "epoch": 17.100694444444443,
1530
+ "grad_norm": 5.71875,
1531
+ "learning_rate": 0.0001479453864669119,
1532
+ "loss": 2.8785,
1533
+ "step": 9850
1534
+ },
1535
+ {
1536
+ "epoch": 17.1875,
1537
+ "grad_norm": 5.4375,
1538
+ "learning_rate": 0.00014746520222589103,
1539
+ "loss": 2.8715,
1540
+ "step": 9900
1541
+ },
1542
+ {
1543
+ "epoch": 17.274305555555557,
1544
+ "grad_norm": 11.8125,
1545
+ "learning_rate": 0.00014698360108777097,
1546
+ "loss": 2.8826,
1547
+ "step": 9950
1548
+ },
1549
+ {
1550
+ "epoch": 17.36111111111111,
1551
+ "grad_norm": 5.96875,
1552
+ "learning_rate": 0.00014650059742896265,
1553
+ "loss": 2.8958,
1554
+ "step": 10000
1555
+ },
1556
+ {
1557
+ "epoch": 17.447916666666668,
1558
+ "grad_norm": 5.75,
1559
+ "learning_rate": 0.00014601620566774415,
1560
+ "loss": 2.8751,
1561
+ "step": 10050
1562
+ },
1563
+ {
1564
+ "epoch": 17.53472222222222,
1565
+ "grad_norm": 4.53125,
1566
+ "learning_rate": 0.00014553044026383014,
1567
+ "loss": 2.8925,
1568
+ "step": 10100
1569
+ },
1570
+ {
1571
+ "epoch": 17.62152777777778,
1572
+ "grad_norm": 5.28125,
1573
+ "learning_rate": 0.0001450433157179403,
1574
+ "loss": 2.8889,
1575
+ "step": 10150
1576
+ },
1577
+ {
1578
+ "epoch": 17.708333333333332,
1579
+ "grad_norm": 6.21875,
1580
+ "learning_rate": 0.00014455484657136642,
1581
+ "loss": 2.8807,
1582
+ "step": 10200
1583
+ },
1584
+ {
1585
+ "epoch": 17.79513888888889,
1586
+ "grad_norm": 5.375,
1587
+ "learning_rate": 0.00014406504740553837,
1588
+ "loss": 2.8836,
1589
+ "step": 10250
1590
+ },
1591
+ {
1592
+ "epoch": 17.881944444444443,
1593
+ "grad_norm": 5.125,
1594
+ "learning_rate": 0.00014357393284158878,
1595
+ "loss": 2.8723,
1596
+ "step": 10300
1597
+ },
1598
+ {
1599
+ "epoch": 17.96875,
1600
+ "grad_norm": 5.3125,
1601
+ "learning_rate": 0.00014308151753991658,
1602
+ "loss": 2.881,
1603
+ "step": 10350
1604
+ },
1605
+ {
1606
+ "epoch": 18.0,
1607
+ "eval_loss": 2.8605997562408447,
1608
+ "eval_runtime": 40.45,
1609
+ "eval_samples_per_second": 92.188,
1610
+ "eval_steps_per_second": 5.785,
1611
+ "step": 10368
1612
+ },
1613
+ {
1614
+ "epoch": 18.055555555555557,
1615
+ "grad_norm": 4.5625,
1616
+ "learning_rate": 0.00014258781619974945,
1617
+ "loss": 2.8781,
1618
+ "step": 10400
1619
+ },
1620
+ {
1621
+ "epoch": 18.14236111111111,
1622
+ "grad_norm": 4.625,
1623
+ "learning_rate": 0.00014209284355870492,
1624
+ "loss": 2.8705,
1625
+ "step": 10450
1626
+ },
1627
+ {
1628
+ "epoch": 18.229166666666668,
1629
+ "grad_norm": 6.09375,
1630
+ "learning_rate": 0.00014159661439235046,
1631
+ "loss": 2.876,
1632
+ "step": 10500
1633
+ },
1634
+ {
1635
+ "epoch": 18.31597222222222,
1636
+ "grad_norm": 4.875,
1637
+ "learning_rate": 0.0001410991435137625,
1638
+ "loss": 2.8918,
1639
+ "step": 10550
1640
+ },
1641
+ {
1642
+ "epoch": 18.40277777777778,
1643
+ "grad_norm": 4.375,
1644
+ "learning_rate": 0.00014060044577308408,
1645
+ "loss": 2.8759,
1646
+ "step": 10600
1647
+ },
1648
+ {
1649
+ "epoch": 18.489583333333332,
1650
+ "grad_norm": 5.90625,
1651
+ "learning_rate": 0.00014010053605708174,
1652
+ "loss": 2.8958,
1653
+ "step": 10650
1654
+ },
1655
+ {
1656
+ "epoch": 18.57638888888889,
1657
+ "grad_norm": 5.125,
1658
+ "learning_rate": 0.000139599429288701,
1659
+ "loss": 2.8763,
1660
+ "step": 10700
1661
+ },
1662
+ {
1663
+ "epoch": 18.663194444444443,
1664
+ "grad_norm": 4.96875,
1665
+ "learning_rate": 0.00013909714042662085,
1666
+ "loss": 2.8905,
1667
+ "step": 10750
1668
+ },
1669
+ {
1670
+ "epoch": 18.75,
1671
+ "grad_norm": 5.625,
1672
+ "learning_rate": 0.00013859368446480743,
1673
+ "loss": 2.8782,
1674
+ "step": 10800
1675
+ },
1676
+ {
1677
+ "epoch": 18.836805555555557,
1678
+ "grad_norm": 5.375,
1679
+ "learning_rate": 0.0001380890764320662,
1680
+ "loss": 2.8834,
1681
+ "step": 10850
1682
+ },
1683
+ {
1684
+ "epoch": 18.92361111111111,
1685
+ "grad_norm": 3.984375,
1686
+ "learning_rate": 0.00013758333139159343,
1687
+ "loss": 2.8705,
1688
+ "step": 10900
1689
+ },
1690
+ {
1691
+ "epoch": 19.0,
1692
+ "eval_loss": 2.8602795600891113,
1693
+ "eval_runtime": 40.4167,
1694
+ "eval_samples_per_second": 92.264,
1695
+ "eval_steps_per_second": 5.79,
1696
+ "step": 10944
1697
+ },
1698
+ {
1699
+ "epoch": 19.010416666666668,
1700
+ "grad_norm": 5.71875,
1701
+ "learning_rate": 0.00013707646444052656,
1702
+ "loss": 2.8757,
1703
+ "step": 10950
1704
+ },
1705
+ {
1706
+ "epoch": 19.09722222222222,
1707
+ "grad_norm": 7.3125,
1708
+ "learning_rate": 0.0001365684907094935,
1709
+ "loss": 2.8753,
1710
+ "step": 11000
1711
+ },
1712
+ {
1713
+ "epoch": 19.18402777777778,
1714
+ "grad_norm": 5.78125,
1715
+ "learning_rate": 0.0001360594253621609,
1716
+ "loss": 2.8632,
1717
+ "step": 11050
1718
+ },
1719
+ {
1720
+ "epoch": 19.270833333333332,
1721
+ "grad_norm": 5.28125,
1722
+ "learning_rate": 0.0001355492835947816,
1723
+ "loss": 2.8771,
1724
+ "step": 11100
1725
+ },
1726
+ {
1727
+ "epoch": 19.35763888888889,
1728
+ "grad_norm": 4.53125,
1729
+ "learning_rate": 0.000135038080635741,
1730
+ "loss": 2.8945,
1731
+ "step": 11150
1732
+ },
1733
+ {
1734
+ "epoch": 19.444444444444443,
1735
+ "grad_norm": 4.25,
1736
+ "learning_rate": 0.00013452583174510237,
1737
+ "loss": 2.87,
1738
+ "step": 11200
1739
+ },
1740
+ {
1741
+ "epoch": 19.53125,
1742
+ "grad_norm": 5.1875,
1743
+ "learning_rate": 0.0001340125522141514,
1744
+ "loss": 2.8948,
1745
+ "step": 11250
1746
+ },
1747
+ {
1748
+ "epoch": 19.618055555555557,
1749
+ "grad_norm": 5.9375,
1750
+ "learning_rate": 0.00013349825736493965,
1751
+ "loss": 2.882,
1752
+ "step": 11300
1753
+ },
1754
+ {
1755
+ "epoch": 19.70486111111111,
1756
+ "grad_norm": 7.0,
1757
+ "learning_rate": 0.00013298296254982733,
1758
+ "loss": 2.8753,
1759
+ "step": 11350
1760
+ },
1761
+ {
1762
+ "epoch": 19.791666666666668,
1763
+ "grad_norm": 4.78125,
1764
+ "learning_rate": 0.00013246668315102487,
1765
+ "loss": 2.8823,
1766
+ "step": 11400
1767
+ },
1768
+ {
1769
+ "epoch": 19.87847222222222,
1770
+ "grad_norm": 4.59375,
1771
+ "learning_rate": 0.00013194943458013375,
1772
+ "loss": 2.8675,
1773
+ "step": 11450
1774
+ },
1775
+ {
1776
+ "epoch": 19.96527777777778,
1777
+ "grad_norm": 6.0,
1778
+ "learning_rate": 0.00013143123227768658,
1779
+ "loss": 2.8765,
1780
+ "step": 11500
1781
+ },
1782
+ {
1783
+ "epoch": 20.0,
1784
+ "eval_loss": 2.8591601848602295,
1785
+ "eval_runtime": 41.4885,
1786
+ "eval_samples_per_second": 89.88,
1787
+ "eval_steps_per_second": 5.64,
1788
+ "step": 11520
1789
+ },
1790
+ {
1791
+ "epoch": 20.052083333333332,
1792
+ "grad_norm": 4.9375,
1793
+ "learning_rate": 0.00013091209171268599,
1794
+ "loss": 2.8735,
1795
+ "step": 11550
1796
+ },
1797
+ {
1798
+ "epoch": 20.13888888888889,
1799
+ "grad_norm": 5.40625,
1800
+ "learning_rate": 0.00013039202838214294,
1801
+ "loss": 2.8698,
1802
+ "step": 11600
1803
+ },
1804
+ {
1805
+ "epoch": 20.225694444444443,
1806
+ "grad_norm": 5.34375,
1807
+ "learning_rate": 0.0001298710578106142,
1808
+ "loss": 2.8702,
1809
+ "step": 11650
1810
+ },
1811
+ {
1812
+ "epoch": 20.3125,
1813
+ "grad_norm": 6.59375,
1814
+ "learning_rate": 0.00012934919554973874,
1815
+ "loss": 2.8871,
1816
+ "step": 11700
1817
+ },
1818
+ {
1819
+ "epoch": 20.399305555555557,
1820
+ "grad_norm": 4.375,
1821
+ "learning_rate": 0.00012882645717777376,
1822
+ "loss": 2.8752,
1823
+ "step": 11750
1824
+ },
1825
+ {
1826
+ "epoch": 20.48611111111111,
1827
+ "grad_norm": 6.78125,
1828
+ "learning_rate": 0.00012830285829912926,
1829
+ "loss": 2.8896,
1830
+ "step": 11800
1831
+ },
1832
+ {
1833
+ "epoch": 20.572916666666668,
1834
+ "grad_norm": 5.21875,
1835
+ "learning_rate": 0.00012777841454390275,
1836
+ "loss": 2.8768,
1837
+ "step": 11850
1838
+ },
1839
+ {
1840
+ "epoch": 20.65972222222222,
1841
+ "grad_norm": 5.59375,
1842
+ "learning_rate": 0.00012725314156741214,
1843
+ "loss": 2.8846,
1844
+ "step": 11900
1845
+ },
1846
+ {
1847
+ "epoch": 20.74652777777778,
1848
+ "grad_norm": 5.25,
1849
+ "learning_rate": 0.00012672705504972884,
1850
+ "loss": 2.873,
1851
+ "step": 11950
1852
+ },
1853
+ {
1854
+ "epoch": 20.833333333333332,
1855
+ "grad_norm": 5.6875,
1856
+ "learning_rate": 0.00012620017069520936,
1857
+ "loss": 2.8809,
1858
+ "step": 12000
1859
+ },
1860
+ {
1861
+ "epoch": 20.92013888888889,
1862
+ "grad_norm": 4.40625,
1863
+ "learning_rate": 0.00012567250423202675,
1864
+ "loss": 2.8656,
1865
+ "step": 12050
1866
+ },
1867
+ {
1868
+ "epoch": 21.0,
1869
+ "eval_loss": 2.857980489730835,
1870
+ "eval_runtime": 41.0572,
1871
+ "eval_samples_per_second": 90.825,
1872
+ "eval_steps_per_second": 5.699,
1873
+ "step": 12096
1874
+ },
1875
+ {
1876
+ "epoch": 21.006944444444443,
1877
+ "grad_norm": 5.5,
1878
+ "learning_rate": 0.00012514407141170104,
1879
+ "loss": 2.8738,
1880
+ "step": 12100
1881
+ },
1882
+ {
1883
+ "epoch": 21.09375,
1884
+ "grad_norm": 4.78125,
1885
+ "learning_rate": 0.00012461488800862887,
1886
+ "loss": 2.8725,
1887
+ "step": 12150
1888
+ },
1889
+ {
1890
+ "epoch": 21.180555555555557,
1891
+ "grad_norm": 8.375,
1892
+ "learning_rate": 0.00012408496981961288,
1893
+ "loss": 2.8628,
1894
+ "step": 12200
1895
+ },
1896
+ {
1897
+ "epoch": 21.26736111111111,
1898
+ "grad_norm": 7.375,
1899
+ "learning_rate": 0.00012355433266338992,
1900
+ "loss": 2.8733,
1901
+ "step": 12250
1902
+ },
1903
+ {
1904
+ "epoch": 21.354166666666668,
1905
+ "grad_norm": 4.65625,
1906
+ "learning_rate": 0.00012302299238015895,
1907
+ "loss": 2.8901,
1908
+ "step": 12300
1909
+ },
1910
+ {
1911
+ "epoch": 21.44097222222222,
1912
+ "grad_norm": 5.0625,
1913
+ "learning_rate": 0.0001224909648311082,
1914
+ "loss": 2.8696,
1915
+ "step": 12350
1916
+ },
1917
+ {
1918
+ "epoch": 21.52777777777778,
1919
+ "grad_norm": 6.09375,
1920
+ "learning_rate": 0.00012195826589794162,
1921
+ "loss": 2.8925,
1922
+ "step": 12400
1923
+ },
1924
+ {
1925
+ "epoch": 21.614583333333332,
1926
+ "grad_norm": 6.0625,
1927
+ "learning_rate": 0.00012142491148240491,
1928
+ "loss": 2.8764,
1929
+ "step": 12450
1930
+ },
1931
+ {
1932
+ "epoch": 21.70138888888889,
1933
+ "grad_norm": 4.90625,
1934
+ "learning_rate": 0.00012089091750581067,
1935
+ "loss": 2.8716,
1936
+ "step": 12500
1937
+ },
1938
+ {
1939
+ "epoch": 21.788194444444443,
1940
+ "grad_norm": 5.9375,
1941
+ "learning_rate": 0.0001203562999085633,
1942
+ "loss": 2.8816,
1943
+ "step": 12550
1944
+ },
1945
+ {
1946
+ "epoch": 21.875,
1947
+ "grad_norm": 5.59375,
1948
+ "learning_rate": 0.00011982107464968298,
1949
+ "loss": 2.8677,
1950
+ "step": 12600
1951
+ },
1952
+ {
1953
+ "epoch": 21.961805555555557,
1954
+ "grad_norm": 5.40625,
1955
+ "learning_rate": 0.00011928525770632946,
1956
+ "loss": 2.8729,
1957
+ "step": 12650
1958
+ },
1959
+ {
1960
+ "epoch": 22.0,
1961
+ "eval_loss": 2.857877016067505,
1962
+ "eval_runtime": 42.4855,
1963
+ "eval_samples_per_second": 87.771,
1964
+ "eval_steps_per_second": 5.508,
1965
+ "step": 12672
1966
+ },
1967
+ {
1968
+ "epoch": 22.04861111111111,
1969
+ "grad_norm": 7.5,
1970
+ "learning_rate": 0.000118748865073325,
1971
+ "loss": 2.8712,
1972
+ "step": 12700
1973
+ },
1974
+ {
1975
+ "epoch": 22.135416666666668,
1976
+ "grad_norm": 5.65625,
1977
+ "learning_rate": 0.00011821191276267684,
1978
+ "loss": 2.867,
1979
+ "step": 12750
1980
+ },
1981
+ {
1982
+ "epoch": 22.22222222222222,
1983
+ "grad_norm": 6.1875,
1984
+ "learning_rate": 0.00011767441680309955,
1985
+ "loss": 2.8635,
1986
+ "step": 12800
1987
+ },
1988
+ {
1989
+ "epoch": 22.30902777777778,
1990
+ "grad_norm": 8.625,
1991
+ "learning_rate": 0.00011713639323953602,
1992
+ "loss": 2.886,
1993
+ "step": 12850
1994
+ },
1995
+ {
1996
+ "epoch": 22.395833333333332,
1997
+ "grad_norm": 6.09375,
1998
+ "learning_rate": 0.00011659785813267905,
1999
+ "loss": 2.872,
2000
+ "step": 12900
2001
+ },
2002
+ {
2003
+ "epoch": 22.48263888888889,
2004
+ "grad_norm": 5.625,
2005
+ "learning_rate": 0.0001160588275584915,
2006
+ "loss": 2.8891,
2007
+ "step": 12950
2008
+ },
2009
+ {
2010
+ "epoch": 22.569444444444443,
2011
+ "grad_norm": 4.59375,
2012
+ "learning_rate": 0.00011551931760772661,
2013
+ "loss": 2.8741,
2014
+ "step": 13000
2015
+ },
2016
+ {
2017
+ "epoch": 22.65625,
2018
+ "grad_norm": 6.65625,
2019
+ "learning_rate": 0.00011497934438544769,
2020
+ "loss": 2.8815,
2021
+ "step": 13050
2022
+ },
2023
+ {
2024
+ "epoch": 22.743055555555557,
2025
+ "grad_norm": 5.3125,
2026
+ "learning_rate": 0.00011443892401054719,
2027
+ "loss": 2.8705,
2028
+ "step": 13100
2029
+ },
2030
+ {
2031
+ "epoch": 22.82986111111111,
2032
+ "grad_norm": 4.96875,
2033
+ "learning_rate": 0.00011389807261526573,
2034
+ "loss": 2.8823,
2035
+ "step": 13150
2036
+ },
2037
+ {
2038
+ "epoch": 22.916666666666668,
2039
+ "grad_norm": 5.5,
2040
+ "learning_rate": 0.00011335680634471035,
2041
+ "loss": 2.8596,
2042
+ "step": 13200
2043
+ },
2044
+ {
2045
+ "epoch": 23.0,
2046
+ "eval_loss": 2.856687545776367,
2047
+ "eval_runtime": 42.2388,
2048
+ "eval_samples_per_second": 88.284,
2049
+ "eval_steps_per_second": 5.54,
2050
+ "step": 13248
2051
+ },
2052
+ {
2053
+ "epoch": 23.00347222222222,
2054
+ "grad_norm": 7.3125,
2055
+ "learning_rate": 0.00011281514135637278,
2056
+ "loss": 2.8712,
2057
+ "step": 13250
2058
+ },
2059
+ {
2060
+ "epoch": 23.09027777777778,
2061
+ "grad_norm": 5.46875,
2062
+ "learning_rate": 0.00011227309381964684,
2063
+ "loss": 2.8741,
2064
+ "step": 13300
2065
+ },
2066
+ {
2067
+ "epoch": 23.177083333333332,
2068
+ "grad_norm": 4.875,
2069
+ "learning_rate": 0.00011173067991534598,
2070
+ "loss": 2.8567,
2071
+ "step": 13350
2072
+ },
2073
+ {
2074
+ "epoch": 23.26388888888889,
2075
+ "grad_norm": 4.59375,
2076
+ "learning_rate": 0.00011118791583522023,
2077
+ "loss": 2.8739,
2078
+ "step": 13400
2079
+ },
2080
+ {
2081
+ "epoch": 23.350694444444443,
2082
+ "grad_norm": 5.96875,
2083
+ "learning_rate": 0.00011064481778147275,
2084
+ "loss": 2.8865,
2085
+ "step": 13450
2086
+ },
2087
+ {
2088
+ "epoch": 23.4375,
2089
+ "grad_norm": 5.6875,
2090
+ "learning_rate": 0.00011010140196627627,
2091
+ "loss": 2.8657,
2092
+ "step": 13500
2093
+ },
2094
+ {
2095
+ "epoch": 23.524305555555557,
2096
+ "grad_norm": 5.15625,
2097
+ "learning_rate": 0.00010955768461128911,
2098
+ "loss": 2.8911,
2099
+ "step": 13550
2100
+ },
2101
+ {
2102
+ "epoch": 23.61111111111111,
2103
+ "grad_norm": 5.0625,
2104
+ "learning_rate": 0.00010901368194717091,
2105
+ "loss": 2.8727,
2106
+ "step": 13600
2107
+ },
2108
+ {
2109
+ "epoch": 23.697916666666668,
2110
+ "grad_norm": 5.34375,
2111
+ "learning_rate": 0.00010846941021309817,
2112
+ "loss": 2.8729,
2113
+ "step": 13650
2114
+ },
2115
+ {
2116
+ "epoch": 23.78472222222222,
2117
+ "grad_norm": 5.53125,
2118
+ "learning_rate": 0.00010792488565627953,
2119
+ "loss": 2.8749,
2120
+ "step": 13700
2121
+ },
2122
+ {
2123
+ "epoch": 23.87152777777778,
2124
+ "grad_norm": 4.84375,
2125
+ "learning_rate": 0.00010738012453147062,
2126
+ "loss": 2.87,
2127
+ "step": 13750
2128
+ },
2129
+ {
2130
+ "epoch": 23.958333333333332,
2131
+ "grad_norm": 6.875,
2132
+ "learning_rate": 0.00010683514310048894,
2133
+ "loss": 2.8713,
2134
+ "step": 13800
2135
+ },
2136
+ {
2137
+ "epoch": 24.0,
2138
+ "eval_loss": 2.856473922729492,
2139
+ "eval_runtime": 40.551,
2140
+ "eval_samples_per_second": 91.958,
2141
+ "eval_steps_per_second": 5.771,
2142
+ "step": 13824
2143
+ },
2144
+ {
2145
+ "epoch": 24.04513888888889,
2146
+ "grad_norm": 7.4375,
2147
+ "learning_rate": 0.00010628995763172851,
2148
+ "loss": 2.8675,
2149
+ "step": 13850
2150
+ },
2151
+ {
2152
+ "epoch": 24.131944444444443,
2153
+ "grad_norm": 5.875,
2154
+ "learning_rate": 0.00010574458439967401,
2155
+ "loss": 2.8666,
2156
+ "step": 13900
2157
+ },
2158
+ {
2159
+ "epoch": 24.21875,
2160
+ "grad_norm": 5.40625,
2161
+ "learning_rate": 0.00010519903968441516,
2162
+ "loss": 2.8586,
2163
+ "step": 13950
2164
+ },
2165
+ {
2166
+ "epoch": 24.305555555555557,
2167
+ "grad_norm": 4.8125,
2168
+ "learning_rate": 0.0001046533397711607,
2169
+ "loss": 2.8836,
2170
+ "step": 14000
2171
+ },
2172
+ {
2173
+ "epoch": 24.39236111111111,
2174
+ "grad_norm": 4.46875,
2175
+ "learning_rate": 0.00010410750094975215,
2176
+ "loss": 2.8711,
2177
+ "step": 14050
2178
+ },
2179
+ {
2180
+ "epoch": 24.479166666666668,
2181
+ "grad_norm": 5.59375,
2182
+ "learning_rate": 0.00010356153951417771,
2183
+ "loss": 2.8866,
2184
+ "step": 14100
2185
+ },
2186
+ {
2187
+ "epoch": 24.56597222222222,
2188
+ "grad_norm": 5.3125,
2189
+ "learning_rate": 0.00010301547176208568,
2190
+ "loss": 2.8723,
2191
+ "step": 14150
2192
+ },
2193
+ {
2194
+ "epoch": 24.65277777777778,
2195
+ "grad_norm": 5.71875,
2196
+ "learning_rate": 0.00010246931399429812,
2197
+ "loss": 2.8754,
2198
+ "step": 14200
2199
+ },
2200
+ {
2201
+ "epoch": 24.739583333333332,
2202
+ "grad_norm": 5.46875,
2203
+ "learning_rate": 0.00010192308251432412,
2204
+ "loss": 2.8733,
2205
+ "step": 14250
2206
+ },
2207
+ {
2208
+ "epoch": 24.82638888888889,
2209
+ "grad_norm": 5.8125,
2210
+ "learning_rate": 0.0001013767936278732,
2211
+ "loss": 2.8821,
2212
+ "step": 14300
2213
+ },
2214
+ {
2215
+ "epoch": 24.913194444444443,
2216
+ "grad_norm": 4.96875,
2217
+ "learning_rate": 0.00010083046364236854,
2218
+ "loss": 2.8564,
2219
+ "step": 14350
2220
+ },
2221
+ {
2222
+ "epoch": 25.0,
2223
+ "grad_norm": 5.90625,
2224
+ "learning_rate": 0.00010028410886646014,
2225
+ "loss": 2.8708,
2226
+ "step": 14400
2227
+ },
2228
+ {
2229
+ "epoch": 25.0,
2230
+ "eval_loss": 2.8556883335113525,
2231
+ "eval_runtime": 39.6238,
2232
+ "eval_samples_per_second": 94.11,
2233
+ "eval_steps_per_second": 5.906,
2234
+ "step": 14400
2235
+ },
2236
+ {
2237
+ "epoch": 25.086805555555557,
2238
+ "grad_norm": 4.46875,
2239
+ "learning_rate": 9.97377456095381e-05,
2240
+ "loss": 2.8732,
2241
+ "step": 14450
2242
+ },
2243
+ {
2244
+ "epoch": 25.17361111111111,
2245
+ "grad_norm": 4.1875,
2246
+ "learning_rate": 9.91913901812456e-05,
2247
+ "loss": 2.8537,
2248
+ "step": 14500
2249
+ },
2250
+ {
2251
+ "epoch": 25.260416666666668,
2252
+ "grad_norm": 3.640625,
2253
+ "learning_rate": 9.864505889099217e-05,
2254
+ "loss": 2.8704,
2255
+ "step": 14550
2256
+ },
2257
+ {
2258
+ "epoch": 25.34722222222222,
2259
+ "grad_norm": 5.5,
2260
+ "learning_rate": 9.809876804746683e-05,
2261
+ "loss": 2.8865,
2262
+ "step": 14600
2263
+ },
2264
+ {
2265
+ "epoch": 25.43402777777778,
2266
+ "grad_norm": 6.6875,
2267
+ "learning_rate": 9.755253395815116e-05,
2268
+ "loss": 2.8648,
2269
+ "step": 14650
2270
+ },
2271
+ {
2272
+ "epoch": 25.520833333333332,
2273
+ "grad_norm": 5.46875,
2274
+ "learning_rate": 9.700637292883252e-05,
2275
+ "loss": 2.8886,
2276
+ "step": 14700
2277
+ },
2278
+ {
2279
+ "epoch": 25.60763888888889,
2280
+ "grad_norm": 4.96875,
2281
+ "learning_rate": 9.646030126311743e-05,
2282
+ "loss": 2.872,
2283
+ "step": 14750
2284
+ },
2285
+ {
2286
+ "epoch": 25.694444444444443,
2287
+ "grad_norm": 7.9375,
2288
+ "learning_rate": 9.591433526194474e-05,
2289
+ "loss": 2.8698,
2290
+ "step": 14800
2291
+ },
2292
+ {
2293
+ "epoch": 25.78125,
2294
+ "grad_norm": 5.0,
2295
+ "learning_rate": 9.536849122309901e-05,
2296
+ "loss": 2.8718,
2297
+ "step": 14850
2298
+ },
2299
+ {
2300
+ "epoch": 25.868055555555557,
2301
+ "grad_norm": 6.1875,
2302
+ "learning_rate": 9.482278544072425e-05,
2303
+ "loss": 2.8712,
2304
+ "step": 14900
2305
+ },
2306
+ {
2307
+ "epoch": 25.95486111111111,
2308
+ "grad_norm": 5.34375,
2309
+ "learning_rate": 9.427723420483717e-05,
2310
+ "loss": 2.8674,
2311
+ "step": 14950
2312
+ },
2313
+ {
2314
+ "epoch": 26.0,
2315
+ "eval_loss": 2.855642080307007,
2316
+ "eval_runtime": 41.248,
2317
+ "eval_samples_per_second": 90.404,
2318
+ "eval_steps_per_second": 5.673,
2319
+ "step": 14976
2320
+ },
2321
+ {
2322
+ "epoch": 26.041666666666668,
2323
+ "grad_norm": 6.46875,
2324
+ "learning_rate": 9.373185380084113e-05,
2325
+ "loss": 2.8681,
2326
+ "step": 15000
2327
+ },
2328
+ {
2329
+ "epoch": 26.12847222222222,
2330
+ "grad_norm": 6.21875,
2331
+ "learning_rate": 9.318666050903988e-05,
2332
+ "loss": 2.8659,
2333
+ "step": 15050
2334
+ },
2335
+ {
2336
+ "epoch": 26.21527777777778,
2337
+ "grad_norm": 5.65625,
2338
+ "learning_rate": 9.264167060415178e-05,
2339
+ "loss": 2.857,
2340
+ "step": 15100
2341
+ },
2342
+ {
2343
+ "epoch": 26.302083333333332,
2344
+ "grad_norm": 4.4375,
2345
+ "learning_rate": 9.209690035482372e-05,
2346
+ "loss": 2.8821,
2347
+ "step": 15150
2348
+ },
2349
+ {
2350
+ "epoch": 26.38888888888889,
2351
+ "grad_norm": 6.0,
2352
+ "learning_rate": 9.155236602314552e-05,
2353
+ "loss": 2.8707,
2354
+ "step": 15200
2355
+ },
2356
+ {
2357
+ "epoch": 26.475694444444443,
2358
+ "grad_norm": 4.53125,
2359
+ "learning_rate": 9.100808386416475e-05,
2360
+ "loss": 2.8819,
2361
+ "step": 15250
2362
+ },
2363
+ {
2364
+ "epoch": 26.5625,
2365
+ "grad_norm": 4.125,
2366
+ "learning_rate": 9.046407012540115e-05,
2367
+ "loss": 2.8716,
2368
+ "step": 15300
2369
+ },
2370
+ {
2371
+ "epoch": 26.649305555555557,
2372
+ "grad_norm": 5.8125,
2373
+ "learning_rate": 8.992034104636183e-05,
2374
+ "loss": 2.8758,
2375
+ "step": 15350
2376
+ },
2377
+ {
2378
+ "epoch": 26.73611111111111,
2379
+ "grad_norm": 4.84375,
2380
+ "learning_rate": 8.937691285805634e-05,
2381
+ "loss": 2.8716,
2382
+ "step": 15400
2383
+ },
2384
+ {
2385
+ "epoch": 26.822916666666668,
2386
+ "grad_norm": 4.8125,
2387
+ "learning_rate": 8.883380178251249e-05,
2388
+ "loss": 2.8792,
2389
+ "step": 15450
2390
+ },
2391
+ {
2392
+ "epoch": 26.90972222222222,
2393
+ "grad_norm": 4.59375,
2394
+ "learning_rate": 8.829102403229163e-05,
2395
+ "loss": 2.8585,
2396
+ "step": 15500
2397
+ },
2398
+ {
2399
+ "epoch": 26.99652777777778,
2400
+ "grad_norm": 7.09375,
2401
+ "learning_rate": 8.774859581000504e-05,
2402
+ "loss": 2.8683,
2403
+ "step": 15550
2404
+ },
2405
+ {
2406
+ "epoch": 27.0,
2407
+ "eval_loss": 2.8553037643432617,
2408
+ "eval_runtime": 41.718,
2409
+ "eval_samples_per_second": 89.386,
2410
+ "eval_steps_per_second": 5.609,
2411
+ "step": 15552
2412
+ },
2413
+ {
2414
+ "epoch": 27.083333333333332,
2415
+ "grad_norm": 4.78125,
2416
+ "learning_rate": 8.720653330783013e-05,
2417
+ "loss": 2.8705,
2418
+ "step": 15600
2419
+ },
2420
+ {
2421
+ "epoch": 27.17013888888889,
2422
+ "grad_norm": 4.5,
2423
+ "learning_rate": 8.666485270702704e-05,
2424
+ "loss": 2.8559,
2425
+ "step": 15650
2426
+ },
2427
+ {
2428
+ "epoch": 27.256944444444443,
2429
+ "grad_norm": 4.03125,
2430
+ "learning_rate": 8.612357017745578e-05,
2431
+ "loss": 2.87,
2432
+ "step": 15700
2433
+ },
2434
+ {
2435
+ "epoch": 27.34375,
2436
+ "grad_norm": 5.65625,
2437
+ "learning_rate": 8.558270187709328e-05,
2438
+ "loss": 2.8804,
2439
+ "step": 15750
2440
+ },
2441
+ {
2442
+ "epoch": 27.430555555555557,
2443
+ "grad_norm": 5.15625,
2444
+ "learning_rate": 8.504226395155132e-05,
2445
+ "loss": 2.8634,
2446
+ "step": 15800
2447
+ },
2448
+ {
2449
+ "epoch": 27.51736111111111,
2450
+ "grad_norm": 5.40625,
2451
+ "learning_rate": 8.450227253359439e-05,
2452
+ "loss": 2.8878,
2453
+ "step": 15850
2454
+ },
2455
+ {
2456
+ "epoch": 27.604166666666668,
2457
+ "grad_norm": 4.15625,
2458
+ "learning_rate": 8.39627437426581e-05,
2459
+ "loss": 2.8713,
2460
+ "step": 15900
2461
+ },
2462
+ {
2463
+ "epoch": 27.69097222222222,
2464
+ "grad_norm": 4.84375,
2465
+ "learning_rate": 8.34236936843682e-05,
2466
+ "loss": 2.8689,
2467
+ "step": 15950
2468
+ },
2469
+ {
2470
+ "epoch": 27.77777777777778,
2471
+ "grad_norm": 5.34375,
2472
+ "learning_rate": 8.28851384500595e-05,
2473
+ "loss": 2.8706,
2474
+ "step": 16000
2475
+ },
2476
+ {
2477
+ "epoch": 27.864583333333332,
2478
+ "grad_norm": 4.3125,
2479
+ "learning_rate": 8.234709411629572e-05,
2480
+ "loss": 2.8689,
2481
+ "step": 16050
2482
+ },
2483
+ {
2484
+ "epoch": 27.95138888888889,
2485
+ "grad_norm": 8.125,
2486
+ "learning_rate": 8.180957674438966e-05,
2487
+ "loss": 2.8677,
2488
+ "step": 16100
2489
+ },
2490
+ {
2491
+ "epoch": 28.0,
2492
+ "eval_loss": 2.8553411960601807,
2493
+ "eval_runtime": 41.9128,
2494
+ "eval_samples_per_second": 88.97,
2495
+ "eval_steps_per_second": 5.583,
2496
+ "step": 16128
2497
+ },
2498
+ {
2499
+ "epoch": 28.038194444444443,
2500
+ "grad_norm": 4.34375,
2501
+ "learning_rate": 8.12726023799235e-05,
2502
+ "loss": 2.8652,
2503
+ "step": 16150
2504
+ },
2505
+ {
2506
+ "epoch": 28.125,
2507
+ "grad_norm": 5.125,
2508
+ "learning_rate": 8.073618705226998e-05,
2509
+ "loss": 2.8667,
2510
+ "step": 16200
2511
+ },
2512
+ {
2513
+ "epoch": 28.211805555555557,
2514
+ "grad_norm": 4.34375,
2515
+ "learning_rate": 8.020034677411386e-05,
2516
+ "loss": 2.8591,
2517
+ "step": 16250
2518
+ },
2519
+ {
2520
+ "epoch": 28.29861111111111,
2521
+ "grad_norm": 4.9375,
2522
+ "learning_rate": 7.966509754097404e-05,
2523
+ "loss": 2.8778,
2524
+ "step": 16300
2525
+ },
2526
+ {
2527
+ "epoch": 28.385416666666668,
2528
+ "grad_norm": 4.0625,
2529
+ "learning_rate": 7.913045533072587e-05,
2530
+ "loss": 2.8716,
2531
+ "step": 16350
2532
+ },
2533
+ {
2534
+ "epoch": 28.47222222222222,
2535
+ "grad_norm": 5.125,
2536
+ "learning_rate": 7.859643610312424e-05,
2537
+ "loss": 2.8786,
2538
+ "step": 16400
2539
+ },
2540
+ {
2541
+ "epoch": 28.55902777777778,
2542
+ "grad_norm": 5.375,
2543
+ "learning_rate": 7.80630557993274e-05,
2544
+ "loss": 2.8746,
2545
+ "step": 16450
2546
+ },
2547
+ {
2548
+ "epoch": 28.645833333333332,
2549
+ "grad_norm": 4.75,
2550
+ "learning_rate": 7.753033034142075e-05,
2551
+ "loss": 2.871,
2552
+ "step": 16500
2553
+ },
2554
+ {
2555
+ "epoch": 28.73263888888889,
2556
+ "grad_norm": 5.09375,
2557
+ "learning_rate": 7.69982756319417e-05,
2558
+ "loss": 2.8704,
2559
+ "step": 16550
2560
+ },
2561
+ {
2562
+ "epoch": 28.819444444444443,
2563
+ "grad_norm": 5.03125,
2564
+ "learning_rate": 7.646690755340504e-05,
2565
+ "loss": 2.8813,
2566
+ "step": 16600
2567
+ },
2568
+ {
2569
+ "epoch": 28.90625,
2570
+ "grad_norm": 4.53125,
2571
+ "learning_rate": 7.59362419678287e-05,
2572
+ "loss": 2.8563,
2573
+ "step": 16650
2574
+ },
2575
+ {
2576
+ "epoch": 28.993055555555557,
2577
+ "grad_norm": 4.53125,
2578
+ "learning_rate": 7.540629471626026e-05,
2579
+ "loss": 2.868,
2580
+ "step": 16700
2581
+ },
2582
+ {
2583
+ "epoch": 29.0,
2584
+ "eval_loss": 2.8549838066101074,
2585
+ "eval_runtime": 40.2288,
2586
+ "eval_samples_per_second": 92.695,
2587
+ "eval_steps_per_second": 5.817,
2588
+ "step": 16704
2589
+ },
2590
+ {
2591
+ "epoch": 29.07986111111111,
2592
+ "grad_norm": 4.90625,
2593
+ "learning_rate": 7.48770816183042e-05,
2594
+ "loss": 2.869,
2595
+ "step": 16750
2596
+ },
2597
+ {
2598
+ "epoch": 29.166666666666668,
2599
+ "grad_norm": 4.1875,
2600
+ "learning_rate": 7.434861847164955e-05,
2601
+ "loss": 2.8525,
2602
+ "step": 16800
2603
+ },
2604
+ {
2605
+ "epoch": 29.25347222222222,
2606
+ "grad_norm": 4.125,
2607
+ "learning_rate": 7.382092105159825e-05,
2608
+ "loss": 2.868,
2609
+ "step": 16850
2610
+ },
2611
+ {
2612
+ "epoch": 29.34027777777778,
2613
+ "grad_norm": 6.125,
2614
+ "learning_rate": 7.329400511059442e-05,
2615
+ "loss": 2.8797,
2616
+ "step": 16900
2617
+ },
2618
+ {
2619
+ "epoch": 29.427083333333332,
2620
+ "grad_norm": 4.71875,
2621
+ "learning_rate": 7.276788637775393e-05,
2622
+ "loss": 2.8629,
2623
+ "step": 16950
2624
+ },
2625
+ {
2626
+ "epoch": 29.51388888888889,
2627
+ "grad_norm": 3.90625,
2628
+ "learning_rate": 7.224258055839509e-05,
2629
+ "loss": 2.8888,
2630
+ "step": 17000
2631
+ },
2632
+ {
2633
+ "epoch": 29.600694444444443,
2634
+ "grad_norm": 4.8125,
2635
+ "learning_rate": 7.171810333356961e-05,
2636
+ "loss": 2.869,
2637
+ "step": 17050
2638
+ },
2639
+ {
2640
+ "epoch": 29.6875,
2641
+ "grad_norm": 5.0625,
2642
+ "learning_rate": 7.119447035959457e-05,
2643
+ "loss": 2.8709,
2644
+ "step": 17100
2645
+ },
2646
+ {
2647
+ "epoch": 29.774305555555557,
2648
+ "grad_norm": 3.828125,
2649
+ "learning_rate": 7.067169726758522e-05,
2650
+ "loss": 2.8669,
2651
+ "step": 17150
2652
+ },
2653
+ {
2654
+ "epoch": 29.86111111111111,
2655
+ "grad_norm": 5.5625,
2656
+ "learning_rate": 7.014979966298808e-05,
2657
+ "loss": 2.8698,
2658
+ "step": 17200
2659
+ },
2660
+ {
2661
+ "epoch": 29.947916666666668,
2662
+ "grad_norm": 3.546875,
2663
+ "learning_rate": 6.962879312511531e-05,
2664
+ "loss": 2.8669,
2665
+ "step": 17250
2666
+ },
2667
+ {
2668
+ "epoch": 30.0,
2669
+ "eval_loss": 2.854860544204712,
2670
+ "eval_runtime": 41.7924,
2671
+ "eval_samples_per_second": 89.227,
2672
+ "eval_steps_per_second": 5.599,
2673
+ "step": 17280
2674
+ },
2675
+ {
2676
+ "epoch": 30.03472222222222,
2677
+ "grad_norm": 5.53125,
2678
+ "learning_rate": 6.910869320667955e-05,
2679
+ "loss": 2.8649,
2680
+ "step": 17300
2681
+ },
2682
+ {
2683
+ "epoch": 30.12152777777778,
2684
+ "grad_norm": 6.15625,
2685
+ "learning_rate": 6.858951543332978e-05,
2686
+ "loss": 2.8648,
2687
+ "step": 17350
2688
+ },
2689
+ {
2690
+ "epoch": 30.208333333333332,
2691
+ "grad_norm": 5.3125,
2692
+ "learning_rate": 6.807127530318771e-05,
2693
+ "loss": 2.8618,
2694
+ "step": 17400
2695
+ },
2696
+ {
2697
+ "epoch": 30.29513888888889,
2698
+ "grad_norm": 4.625,
2699
+ "learning_rate": 6.755398828638512e-05,
2700
+ "loss": 2.8748,
2701
+ "step": 17450
2702
+ },
2703
+ {
2704
+ "epoch": 30.381944444444443,
2705
+ "grad_norm": 4.6875,
2706
+ "learning_rate": 6.703766982460231e-05,
2707
+ "loss": 2.8702,
2708
+ "step": 17500
2709
+ },
2710
+ {
2711
+ "epoch": 30.46875,
2712
+ "grad_norm": 4.5,
2713
+ "learning_rate": 6.652233533060683e-05,
2714
+ "loss": 2.8766,
2715
+ "step": 17550
2716
+ },
2717
+ {
2718
+ "epoch": 30.555555555555557,
2719
+ "grad_norm": 4.65625,
2720
+ "learning_rate": 6.600800018779356e-05,
2721
+ "loss": 2.8766,
2722
+ "step": 17600
2723
+ },
2724
+ {
2725
+ "epoch": 30.64236111111111,
2726
+ "grad_norm": 5.53125,
2727
+ "learning_rate": 6.549467974972552e-05,
2728
+ "loss": 2.8674,
2729
+ "step": 17650
2730
+ },
2731
+ {
2732
+ "epoch": 30.729166666666668,
2733
+ "grad_norm": 5.28125,
2734
+ "learning_rate": 6.498238933967544e-05,
2735
+ "loss": 2.868,
2736
+ "step": 17700
2737
+ },
2738
+ {
2739
+ "epoch": 30.81597222222222,
2740
+ "grad_norm": 4.09375,
2741
+ "learning_rate": 6.44711442501684e-05,
2742
+ "loss": 2.8798,
2743
+ "step": 17750
2744
+ },
2745
+ {
2746
+ "epoch": 30.90277777777778,
2747
+ "grad_norm": 6.03125,
2748
+ "learning_rate": 6.396095974252534e-05,
2749
+ "loss": 2.8578,
2750
+ "step": 17800
2751
+ },
2752
+ {
2753
+ "epoch": 30.989583333333332,
2754
+ "grad_norm": 5.59375,
2755
+ "learning_rate": 6.345185104640747e-05,
2756
+ "loss": 2.8672,
2757
+ "step": 17850
2758
+ },
2759
+ {
2760
+ "epoch": 31.0,
2761
+ "eval_loss": 2.8543925285339355,
2762
+ "eval_runtime": 41.2327,
2763
+ "eval_samples_per_second": 90.438,
2764
+ "eval_steps_per_second": 5.675,
2765
+ "step": 17856
2766
+ },
2767
+ {
2768
+ "epoch": 31.07638888888889,
2769
+ "grad_norm": 4.625,
2770
+ "learning_rate": 6.294383335936167e-05,
2771
+ "loss": 2.87,
2772
+ "step": 17900
2773
+ },
2774
+ {
2775
+ "epoch": 31.163194444444443,
2776
+ "grad_norm": 3.78125,
2777
+ "learning_rate": 6.24369218463667e-05,
2778
+ "loss": 2.8516,
2779
+ "step": 17950
2780
+ },
2781
+ {
2782
+ "epoch": 31.25,
2783
+ "grad_norm": 5.4375,
2784
+ "learning_rate": 6.193113163938075e-05,
2785
+ "loss": 2.8673,
2786
+ "step": 18000
2787
+ },
2788
+ {
2789
+ "epoch": 31.336805555555557,
2790
+ "grad_norm": 4.4375,
2791
+ "learning_rate": 6.14264778368895e-05,
2792
+ "loss": 2.8794,
2793
+ "step": 18050
2794
+ },
2795
+ {
2796
+ "epoch": 31.42361111111111,
2797
+ "grad_norm": 5.125,
2798
+ "learning_rate": 6.092297550345554e-05,
2799
+ "loss": 2.8634,
2800
+ "step": 18100
2801
+ },
2802
+ {
2803
+ "epoch": 31.510416666666668,
2804
+ "grad_norm": 5.46875,
2805
+ "learning_rate": 6.0420639669268544e-05,
2806
+ "loss": 2.8904,
2807
+ "step": 18150
2808
+ },
2809
+ {
2810
+ "epoch": 31.59722222222222,
2811
+ "grad_norm": 4.21875,
2812
+ "learning_rate": 5.991948532969685e-05,
2813
+ "loss": 2.8651,
2814
+ "step": 18200
2815
+ },
2816
+ {
2817
+ "epoch": 31.68402777777778,
2818
+ "grad_norm": 4.6875,
2819
+ "learning_rate": 5.9419527444839515e-05,
2820
+ "loss": 2.8727,
2821
+ "step": 18250
2822
+ },
2823
+ {
2824
+ "epoch": 31.770833333333332,
2825
+ "grad_norm": 3.765625,
2826
+ "learning_rate": 5.8920780939079955e-05,
2827
+ "loss": 2.8645,
2828
+ "step": 18300
2829
+ },
2830
+ {
2831
+ "epoch": 31.85763888888889,
2832
+ "grad_norm": 6.28125,
2833
+ "learning_rate": 5.8423260700640417e-05,
2834
+ "loss": 2.8713,
2835
+ "step": 18350
2836
+ },
2837
+ {
2838
+ "epoch": 31.944444444444443,
2839
+ "grad_norm": 6.9375,
2840
+ "learning_rate": 5.792698158113742e-05,
2841
+ "loss": 2.8634,
2842
+ "step": 18400
2843
+ },
2844
+ {
2845
+ "epoch": 32.0,
2846
+ "eval_loss": 2.8544044494628906,
2847
+ "eval_runtime": 40.4905,
2848
+ "eval_samples_per_second": 92.096,
2849
+ "eval_steps_per_second": 5.779,
2850
+ "step": 18432
2851
+ },
2852
+ {
2853
+ "epoch": 32.03125,
2854
+ "grad_norm": 3.84375,
2855
+ "learning_rate": 5.743195839513852e-05,
2856
+ "loss": 2.8657,
2857
+ "step": 18450
2858
+ },
2859
+ {
2860
+ "epoch": 32.11805555555556,
2861
+ "grad_norm": 4.65625,
2862
+ "learning_rate": 5.693820591971996e-05,
2863
+ "loss": 2.8633,
2864
+ "step": 18500
2865
+ },
2866
+ {
2867
+ "epoch": 32.204861111111114,
2868
+ "grad_norm": 5.25,
2869
+ "learning_rate": 5.644573889402589e-05,
2870
+ "loss": 2.8595,
2871
+ "step": 18550
2872
+ },
2873
+ {
2874
+ "epoch": 32.291666666666664,
2875
+ "grad_norm": 4.8125,
2876
+ "learning_rate": 5.5954572018827846e-05,
2877
+ "loss": 2.8737,
2878
+ "step": 18600
2879
+ },
2880
+ {
2881
+ "epoch": 32.37847222222222,
2882
+ "grad_norm": 5.46875,
2883
+ "learning_rate": 5.5464719956086396e-05,
2884
+ "loss": 2.8722,
2885
+ "step": 18650
2886
+ },
2887
+ {
2888
+ "epoch": 32.46527777777778,
2889
+ "grad_norm": 5.15625,
2890
+ "learning_rate": 5.49761973285132e-05,
2891
+ "loss": 2.871,
2892
+ "step": 18700
2893
+ },
2894
+ {
2895
+ "epoch": 32.552083333333336,
2896
+ "grad_norm": 4.1875,
2897
+ "learning_rate": 5.4489018719134654e-05,
2898
+ "loss": 2.8801,
2899
+ "step": 18750
2900
+ },
2901
+ {
2902
+ "epoch": 32.638888888888886,
2903
+ "grad_norm": 4.875,
2904
+ "learning_rate": 5.400319867085633e-05,
2905
+ "loss": 2.8668,
2906
+ "step": 18800
2907
+ },
2908
+ {
2909
+ "epoch": 32.72569444444444,
2910
+ "grad_norm": 4.8125,
2911
+ "learning_rate": 5.3518751686029134e-05,
2912
+ "loss": 2.8673,
2913
+ "step": 18850
2914
+ },
2915
+ {
2916
+ "epoch": 32.8125,
2917
+ "grad_norm": 3.828125,
2918
+ "learning_rate": 5.303569222601626e-05,
2919
+ "loss": 2.875,
2920
+ "step": 18900
2921
+ },
2922
+ {
2923
+ "epoch": 32.89930555555556,
2924
+ "grad_norm": 4.0625,
2925
+ "learning_rate": 5.25540347107615e-05,
2926
+ "loss": 2.8596,
2927
+ "step": 18950
2928
+ },
2929
+ {
2930
+ "epoch": 32.986111111111114,
2931
+ "grad_norm": 4.25,
2932
+ "learning_rate": 5.207379351835875e-05,
2933
+ "loss": 2.8683,
2934
+ "step": 19000
2935
+ },
2936
+ {
2937
+ "epoch": 33.0,
2938
+ "eval_loss": 2.854464054107666,
2939
+ "eval_runtime": 40.2584,
2940
+ "eval_samples_per_second": 92.627,
2941
+ "eval_steps_per_second": 5.812,
2942
+ "step": 19008
2943
+ },
2944
+ {
2945
+ "epoch": 33.072916666666664,
2946
+ "grad_norm": 4.5,
2947
+ "learning_rate": 5.1594982984622906e-05,
2948
+ "loss": 2.8657,
2949
+ "step": 19050
2950
+ },
2951
+ {
2952
+ "epoch": 33.15972222222222,
2953
+ "grad_norm": 5.78125,
2954
+ "learning_rate": 5.1117617402661865e-05,
2955
+ "loss": 2.8538,
2956
+ "step": 19100
2957
+ },
2958
+ {
2959
+ "epoch": 33.24652777777778,
2960
+ "grad_norm": 4.25,
2961
+ "learning_rate": 5.064171102244985e-05,
2962
+ "loss": 2.8671,
2963
+ "step": 19150
2964
+ },
2965
+ {
2966
+ "epoch": 33.333333333333336,
2967
+ "grad_norm": 5.125,
2968
+ "learning_rate": 5.0167278050402075e-05,
2969
+ "loss": 2.879,
2970
+ "step": 19200
2971
+ },
2972
+ {
2973
+ "epoch": 33.420138888888886,
2974
+ "grad_norm": 3.890625,
2975
+ "learning_rate": 4.9694332648950536e-05,
2976
+ "loss": 2.8637,
2977
+ "step": 19250
2978
+ },
2979
+ {
2980
+ "epoch": 33.50694444444444,
2981
+ "grad_norm": 3.71875,
2982
+ "learning_rate": 4.9222888936121494e-05,
2983
+ "loss": 2.8891,
2984
+ "step": 19300
2985
+ },
2986
+ {
2987
+ "epoch": 33.59375,
2988
+ "grad_norm": 3.578125,
2989
+ "learning_rate": 4.875296098511365e-05,
2990
+ "loss": 2.864,
2991
+ "step": 19350
2992
+ },
2993
+ {
2994
+ "epoch": 33.68055555555556,
2995
+ "grad_norm": 4.9375,
2996
+ "learning_rate": 4.828456282387859e-05,
2997
+ "loss": 2.8731,
2998
+ "step": 19400
2999
+ },
3000
+ {
3001
+ "epoch": 33.767361111111114,
3002
+ "grad_norm": 4.3125,
3003
+ "learning_rate": 4.781770843470144e-05,
3004
+ "loss": 2.8677,
3005
+ "step": 19450
3006
+ },
3007
+ {
3008
+ "epoch": 33.854166666666664,
3009
+ "grad_norm": 5.15625,
3010
+ "learning_rate": 4.735241175378386e-05,
3011
+ "loss": 2.8649,
3012
+ "step": 19500
3013
+ },
3014
+ {
3015
+ "epoch": 33.94097222222222,
3016
+ "grad_norm": 4.15625,
3017
+ "learning_rate": 4.688868667082794e-05,
3018
+ "loss": 2.8629,
3019
+ "step": 19550
3020
+ },
3021
+ {
3022
+ "epoch": 34.0,
3023
+ "eval_loss": 2.8541414737701416,
3024
+ "eval_runtime": 39.773,
3025
+ "eval_samples_per_second": 93.757,
3026
+ "eval_steps_per_second": 5.883,
3027
+ "step": 19584
3028
+ },
3029
+ {
3030
+ "epoch": 34.02777777777778,
3031
+ "grad_norm": 4.09375,
3032
+ "learning_rate": 4.642654702862157e-05,
3033
+ "loss": 2.8661,
3034
+ "step": 19600
3035
+ },
3036
+ {
3037
+ "epoch": 34.114583333333336,
3038
+ "grad_norm": 3.53125,
3039
+ "learning_rate": 4.596600662262508e-05,
3040
+ "loss": 2.8641,
3041
+ "step": 19650
3042
+ },
3043
+ {
3044
+ "epoch": 34.201388888888886,
3045
+ "grad_norm": 4.46875,
3046
+ "learning_rate": 4.55070792005597e-05,
3047
+ "loss": 2.8574,
3048
+ "step": 19700
3049
+ },
3050
+ {
3051
+ "epoch": 34.28819444444444,
3052
+ "grad_norm": 3.828125,
3053
+ "learning_rate": 4.5049778461996926e-05,
3054
+ "loss": 2.8735,
3055
+ "step": 19750
3056
+ },
3057
+ {
3058
+ "epoch": 34.375,
3059
+ "grad_norm": 6.65625,
3060
+ "learning_rate": 4.459411805794976e-05,
3061
+ "loss": 2.8731,
3062
+ "step": 19800
3063
+ },
3064
+ {
3065
+ "epoch": 34.46180555555556,
3066
+ "grad_norm": 4.25,
3067
+ "learning_rate": 4.414011159046495e-05,
3068
+ "loss": 2.8719,
3069
+ "step": 19850
3070
+ },
3071
+ {
3072
+ "epoch": 34.548611111111114,
3073
+ "grad_norm": 3.90625,
3074
+ "learning_rate": 4.368777261221737e-05,
3075
+ "loss": 2.8769,
3076
+ "step": 19900
3077
+ },
3078
+ {
3079
+ "epoch": 34.635416666666664,
3080
+ "grad_norm": 4.0625,
3081
+ "learning_rate": 4.323711462610495e-05,
3082
+ "loss": 2.8679,
3083
+ "step": 19950
3084
+ },
3085
+ {
3086
+ "epoch": 34.72222222222222,
3087
+ "grad_norm": 3.953125,
3088
+ "learning_rate": 4.278815108484602e-05,
3089
+ "loss": 2.8681,
3090
+ "step": 20000
3091
+ },
3092
+ {
3093
+ "epoch": 34.80902777777778,
3094
+ "grad_norm": 4.09375,
3095
+ "learning_rate": 4.234089539057745e-05,
3096
+ "loss": 2.8744,
3097
+ "step": 20050
3098
+ },
3099
+ {
3100
+ "epoch": 34.895833333333336,
3101
+ "grad_norm": 4.03125,
3102
+ "learning_rate": 4.1895360894454774e-05,
3103
+ "loss": 2.8615,
3104
+ "step": 20100
3105
+ },
3106
+ {
3107
+ "epoch": 34.982638888888886,
3108
+ "grad_norm": 4.21875,
3109
+ "learning_rate": 4.1451560896253515e-05,
3110
+ "loss": 2.8641,
3111
+ "step": 20150
3112
+ },
3113
+ {
3114
+ "epoch": 35.0,
3115
+ "eval_loss": 2.8540618419647217,
3116
+ "eval_runtime": 41.6189,
3117
+ "eval_samples_per_second": 89.599,
3118
+ "eval_steps_per_second": 5.622,
3119
+ "step": 20160
3120
+ },
3121
+ {
3122
+ "epoch": 35.06944444444444,
3123
+ "grad_norm": 4.1875,
3124
+ "learning_rate": 4.100950864397223e-05,
3125
+ "loss": 2.8629,
3126
+ "step": 20200
3127
+ },
3128
+ {
3129
+ "epoch": 35.15625,
3130
+ "grad_norm": 3.484375,
3131
+ "learning_rate": 4.056921733343704e-05,
3132
+ "loss": 2.8579,
3133
+ "step": 20250
3134
+ },
3135
+ {
3136
+ "epoch": 35.24305555555556,
3137
+ "grad_norm": 5.28125,
3138
+ "learning_rate": 4.013070010790759e-05,
3139
+ "loss": 2.8641,
3140
+ "step": 20300
3141
+ },
3142
+ {
3143
+ "epoch": 35.329861111111114,
3144
+ "grad_norm": 4.25,
3145
+ "learning_rate": 3.9693970057684984e-05,
3146
+ "loss": 2.8801,
3147
+ "step": 20350
3148
+ },
3149
+ {
3150
+ "epoch": 35.416666666666664,
3151
+ "grad_norm": 4.84375,
3152
+ "learning_rate": 3.9259040219720645e-05,
3153
+ "loss": 2.8614,
3154
+ "step": 20400
3155
+ },
3156
+ {
3157
+ "epoch": 35.50347222222222,
3158
+ "grad_norm": 5.21875,
3159
+ "learning_rate": 3.882592357722761e-05,
3160
+ "loss": 2.8902,
3161
+ "step": 20450
3162
+ },
3163
+ {
3164
+ "epoch": 35.59027777777778,
3165
+ "grad_norm": 4.71875,
3166
+ "learning_rate": 3.839463305929247e-05,
3167
+ "loss": 2.8626,
3168
+ "step": 20500
3169
+ },
3170
+ {
3171
+ "epoch": 35.677083333333336,
3172
+ "grad_norm": 4.3125,
3173
+ "learning_rate": 3.7965181540489794e-05,
3174
+ "loss": 2.8741,
3175
+ "step": 20550
3176
+ },
3177
+ {
3178
+ "epoch": 35.763888888888886,
3179
+ "grad_norm": 3.46875,
3180
+ "learning_rate": 3.753758184049764e-05,
3181
+ "loss": 2.8656,
3182
+ "step": 20600
3183
+ },
3184
+ {
3185
+ "epoch": 35.85069444444444,
3186
+ "grad_norm": 5.03125,
3187
+ "learning_rate": 3.7111846723714916e-05,
3188
+ "loss": 2.8661,
3189
+ "step": 20650
3190
+ },
3191
+ {
3192
+ "epoch": 35.9375,
3193
+ "grad_norm": 3.25,
3194
+ "learning_rate": 3.668798889888022e-05,
3195
+ "loss": 2.8597,
3196
+ "step": 20700
3197
+ },
3198
+ {
3199
+ "epoch": 36.0,
3200
+ "eval_loss": 2.853997230529785,
3201
+ "eval_runtime": 41.7365,
3202
+ "eval_samples_per_second": 89.346,
3203
+ "eval_steps_per_second": 5.607,
3204
+ "step": 20736
3205
+ },
3206
+ {
3207
+ "epoch": 36.02430555555556,
3208
+ "grad_norm": 4.5,
3209
+ "learning_rate": 3.626602101869281e-05,
3210
+ "loss": 2.8674,
3211
+ "step": 20750
3212
+ },
3213
+ {
3214
+ "epoch": 36.111111111111114,
3215
+ "grad_norm": 5.15625,
3216
+ "learning_rate": 3.5845955679434426e-05,
3217
+ "loss": 2.8631,
3218
+ "step": 20800
3219
+ },
3220
+ {
3221
+ "epoch": 36.197916666666664,
3222
+ "grad_norm": 3.6875,
3223
+ "learning_rate": 3.542780542059373e-05,
3224
+ "loss": 2.8576,
3225
+ "step": 20850
3226
+ },
3227
+ {
3228
+ "epoch": 36.28472222222222,
3229
+ "grad_norm": 3.515625,
3230
+ "learning_rate": 3.501158272449155e-05,
3231
+ "loss": 2.8715,
3232
+ "step": 20900
3233
+ },
3234
+ {
3235
+ "epoch": 36.37152777777778,
3236
+ "grad_norm": 3.4375,
3237
+ "learning_rate": 3.45973000159088e-05,
3238
+ "loss": 2.8754,
3239
+ "step": 20950
3240
+ },
3241
+ {
3242
+ "epoch": 36.458333333333336,
3243
+ "grad_norm": 3.671875,
3244
+ "learning_rate": 3.418496966171498e-05,
3245
+ "loss": 2.8721,
3246
+ "step": 21000
3247
+ },
3248
+ {
3249
+ "epoch": 36.545138888888886,
3250
+ "grad_norm": 3.84375,
3251
+ "learning_rate": 3.377460397049951e-05,
3252
+ "loss": 2.8741,
3253
+ "step": 21050
3254
+ },
3255
+ {
3256
+ "epoch": 36.63194444444444,
3257
+ "grad_norm": 4.3125,
3258
+ "learning_rate": 3.336621519220404e-05,
3259
+ "loss": 2.8717,
3260
+ "step": 21100
3261
+ },
3262
+ {
3263
+ "epoch": 36.71875,
3264
+ "grad_norm": 3.625,
3265
+ "learning_rate": 3.295981551775679e-05,
3266
+ "loss": 2.8655,
3267
+ "step": 21150
3268
+ },
3269
+ {
3270
+ "epoch": 36.80555555555556,
3271
+ "grad_norm": 3.46875,
3272
+ "learning_rate": 3.255541707870874e-05,
3273
+ "loss": 2.8748,
3274
+ "step": 21200
3275
+ },
3276
+ {
3277
+ "epoch": 36.892361111111114,
3278
+ "grad_norm": 3.640625,
3279
+ "learning_rate": 3.2153031946871427e-05,
3280
+ "loss": 2.8598,
3281
+ "step": 21250
3282
+ },
3283
+ {
3284
+ "epoch": 36.979166666666664,
3285
+ "grad_norm": 3.875,
3286
+ "learning_rate": 3.1752672133956596e-05,
3287
+ "loss": 2.8632,
3288
+ "step": 21300
3289
+ },
3290
+ {
3291
+ "epoch": 37.0,
3292
+ "eval_loss": 2.854156017303467,
3293
+ "eval_runtime": 40.6449,
3294
+ "eval_samples_per_second": 91.746,
3295
+ "eval_steps_per_second": 5.757,
3296
+ "step": 21312
3297
+ },
3298
+ {
3299
+ "epoch": 37.06597222222222,
3300
+ "grad_norm": 4.125,
3301
+ "learning_rate": 3.135434959121756e-05,
3302
+ "loss": 2.8613,
3303
+ "step": 21350
3304
+ },
3305
+ {
3306
+ "epoch": 37.15277777777778,
3307
+ "grad_norm": 5.53125,
3308
+ "learning_rate": 3.095807620909257e-05,
3309
+ "loss": 2.859,
3310
+ "step": 21400
3311
+ },
3312
+ {
3313
+ "epoch": 37.239583333333336,
3314
+ "grad_norm": 4.15625,
3315
+ "learning_rate": 3.0563863816849795e-05,
3316
+ "loss": 2.8618,
3317
+ "step": 21450
3318
+ },
3319
+ {
3320
+ "epoch": 37.326388888888886,
3321
+ "grad_norm": 4.0625,
3322
+ "learning_rate": 3.017172418223424e-05,
3323
+ "loss": 2.8817,
3324
+ "step": 21500
3325
+ },
3326
+ {
3327
+ "epoch": 37.41319444444444,
3328
+ "grad_norm": 3.953125,
3329
+ "learning_rate": 2.9781669011116364e-05,
3330
+ "loss": 2.8609,
3331
+ "step": 21550
3332
+ },
3333
+ {
3334
+ "epoch": 37.5,
3335
+ "grad_norm": 3.96875,
3336
+ "learning_rate": 2.939370994714278e-05,
3337
+ "loss": 2.8872,
3338
+ "step": 21600
3339
+ },
3340
+ {
3341
+ "epoch": 37.58680555555556,
3342
+ "grad_norm": 4.09375,
3343
+ "learning_rate": 2.90078585713886e-05,
3344
+ "loss": 2.864,
3345
+ "step": 21650
3346
+ },
3347
+ {
3348
+ "epoch": 37.673611111111114,
3349
+ "grad_norm": 3.421875,
3350
+ "learning_rate": 2.8624126402011798e-05,
3351
+ "loss": 2.8757,
3352
+ "step": 21700
3353
+ },
3354
+ {
3355
+ "epoch": 37.760416666666664,
3356
+ "grad_norm": 3.40625,
3357
+ "learning_rate": 2.8242524893909162e-05,
3358
+ "loss": 2.8623,
3359
+ "step": 21750
3360
+ },
3361
+ {
3362
+ "epoch": 37.84722222222222,
3363
+ "grad_norm": 3.28125,
3364
+ "learning_rate": 2.7863065438374748e-05,
3365
+ "loss": 2.8695,
3366
+ "step": 21800
3367
+ },
3368
+ {
3369
+ "epoch": 37.93402777777778,
3370
+ "grad_norm": 4.4375,
3371
+ "learning_rate": 2.7485759362759378e-05,
3372
+ "loss": 2.8596,
3373
+ "step": 21850
3374
+ },
3375
+ {
3376
+ "epoch": 38.0,
3377
+ "eval_loss": 2.8540520668029785,
3378
+ "eval_runtime": 41.5194,
3379
+ "eval_samples_per_second": 89.813,
3380
+ "eval_steps_per_second": 5.636,
3381
+ "step": 21888
3382
+ },
3383
+ {
3384
+ "epoch": 38.020833333333336,
3385
+ "grad_norm": 3.25,
3386
+ "learning_rate": 2.7110617930132877e-05,
3387
+ "loss": 2.8658,
3388
+ "step": 21900
3389
+ },
3390
+ {
3391
+ "epoch": 38.107638888888886,
3392
+ "grad_norm": 3.1875,
3393
+ "learning_rate": 2.673765233894755e-05,
3394
+ "loss": 2.8632,
3395
+ "step": 21950
3396
+ },
3397
+ {
3398
+ "epoch": 38.19444444444444,
3399
+ "grad_norm": 3.765625,
3400
+ "learning_rate": 2.6366873722704265e-05,
3401
+ "loss": 2.8583,
3402
+ "step": 22000
3403
+ },
3404
+ {
3405
+ "epoch": 38.28125,
3406
+ "grad_norm": 3.640625,
3407
+ "learning_rate": 2.599829314961967e-05,
3408
+ "loss": 2.8678,
3409
+ "step": 22050
3410
+ },
3411
+ {
3412
+ "epoch": 38.36805555555556,
3413
+ "grad_norm": 3.609375,
3414
+ "learning_rate": 2.5631921622296128e-05,
3415
+ "loss": 2.8777,
3416
+ "step": 22100
3417
+ },
3418
+ {
3419
+ "epoch": 38.454861111111114,
3420
+ "grad_norm": 3.15625,
3421
+ "learning_rate": 2.526777007739316e-05,
3422
+ "loss": 2.8671,
3423
+ "step": 22150
3424
+ },
3425
+ {
3426
+ "epoch": 38.541666666666664,
3427
+ "grad_norm": 3.53125,
3428
+ "learning_rate": 2.4905849385300883e-05,
3429
+ "loss": 2.8782,
3430
+ "step": 22200
3431
+ },
3432
+ {
3433
+ "epoch": 38.62847222222222,
3434
+ "grad_norm": 3.46875,
3435
+ "learning_rate": 2.4546170349815666e-05,
3436
+ "loss": 2.8699,
3437
+ "step": 22250
3438
+ },
3439
+ {
3440
+ "epoch": 38.71527777777778,
3441
+ "grad_norm": 3.8125,
3442
+ "learning_rate": 2.418874370781754e-05,
3443
+ "loss": 2.8658,
3444
+ "step": 22300
3445
+ },
3446
+ {
3447
+ "epoch": 38.802083333333336,
3448
+ "grad_norm": 3.9375,
3449
+ "learning_rate": 2.3833580128949762e-05,
3450
+ "loss": 2.8749,
3451
+ "step": 22350
3452
+ },
3453
+ {
3454
+ "epoch": 38.888888888888886,
3455
+ "grad_norm": 3.75,
3456
+ "learning_rate": 2.3480690215300105e-05,
3457
+ "loss": 2.8573,
3458
+ "step": 22400
3459
+ },
3460
+ {
3461
+ "epoch": 38.97569444444444,
3462
+ "grad_norm": 4.75,
3463
+ "learning_rate": 2.313008450108468e-05,
3464
+ "loss": 2.8656,
3465
+ "step": 22450
3466
+ },
3467
+ {
3468
+ "epoch": 39.0,
3469
+ "eval_loss": 2.854092597961426,
3470
+ "eval_runtime": 41.0282,
3471
+ "eval_samples_per_second": 90.889,
3472
+ "eval_steps_per_second": 5.703,
3473
+ "step": 22464
3474
+ },
3475
+ {
3476
+ "epoch": 39.0625,
3477
+ "grad_norm": 3.40625,
3478
+ "learning_rate": 2.278177345233323e-05,
3479
+ "loss": 2.8622,
3480
+ "step": 22500
3481
+ },
3482
+ {
3483
+ "epoch": 39.14930555555556,
3484
+ "grad_norm": 3.625,
3485
+ "learning_rate": 2.2435767466576863e-05,
3486
+ "loss": 2.8578,
3487
+ "step": 22550
3488
+ },
3489
+ {
3490
+ "epoch": 39.236111111111114,
3491
+ "grad_norm": 4.15625,
3492
+ "learning_rate": 2.209207687253746e-05,
3493
+ "loss": 2.8602,
3494
+ "step": 22600
3495
+ },
3496
+ {
3497
+ "epoch": 39.322916666666664,
3498
+ "grad_norm": 2.921875,
3499
+ "learning_rate": 2.1750711929819723e-05,
3500
+ "loss": 2.8825,
3501
+ "step": 22650
3502
+ },
3503
+ {
3504
+ "epoch": 39.40972222222222,
3505
+ "grad_norm": 3.421875,
3506
+ "learning_rate": 2.1411682828604452e-05,
3507
+ "loss": 2.8618,
3508
+ "step": 22700
3509
+ },
3510
+ {
3511
+ "epoch": 39.49652777777778,
3512
+ "grad_norm": 3.6875,
3513
+ "learning_rate": 2.1074999689344755e-05,
3514
+ "loss": 2.8834,
3515
+ "step": 22750
3516
+ },
3517
+ {
3518
+ "epoch": 39.583333333333336,
3519
+ "grad_norm": 4.46875,
3520
+ "learning_rate": 2.0740672562463602e-05,
3521
+ "loss": 2.8664,
3522
+ "step": 22800
3523
+ },
3524
+ {
3525
+ "epoch": 39.670138888888886,
3526
+ "grad_norm": 3.5625,
3527
+ "learning_rate": 2.0408711428054195e-05,
3528
+ "loss": 2.8771,
3529
+ "step": 22850
3530
+ },
3531
+ {
3532
+ "epoch": 39.75694444444444,
3533
+ "grad_norm": 3.390625,
3534
+ "learning_rate": 2.0079126195581612e-05,
3535
+ "loss": 2.8629,
3536
+ "step": 22900
3537
+ },
3538
+ {
3539
+ "epoch": 39.84375,
3540
+ "grad_norm": 4.28125,
3541
+ "learning_rate": 1.9751926703587353e-05,
3542
+ "loss": 2.867,
3543
+ "step": 22950
3544
+ },
3545
+ {
3546
+ "epoch": 39.93055555555556,
3547
+ "grad_norm": 3.390625,
3548
+ "learning_rate": 1.9427122719395452e-05,
3549
+ "loss": 2.8591,
3550
+ "step": 23000
3551
+ },
3552
+ {
3553
+ "epoch": 40.0,
3554
+ "eval_loss": 2.85404634475708,
3555
+ "eval_runtime": 42.4648,
3556
+ "eval_samples_per_second": 87.814,
3557
+ "eval_steps_per_second": 5.51,
3558
+ "step": 23040
3559
+ },
3560
+ {
3561
+ "epoch": 40.017361111111114,
3562
+ "grad_norm": 4.71875,
3563
+ "learning_rate": 1.9104723938821012e-05,
3564
+ "loss": 2.8661,
3565
+ "step": 23050
3566
+ },
3567
+ {
3568
+ "epoch": 40.104166666666664,
3569
+ "grad_norm": 3.375,
3570
+ "learning_rate": 1.8784739985880628e-05,
3571
+ "loss": 2.8613,
3572
+ "step": 23100
3573
+ },
3574
+ {
3575
+ "epoch": 40.19097222222222,
3576
+ "grad_norm": 3.078125,
3577
+ "learning_rate": 1.8467180412505313e-05,
3578
+ "loss": 2.8565,
3579
+ "step": 23150
3580
+ },
3581
+ {
3582
+ "epoch": 40.27777777777778,
3583
+ "grad_norm": 3.171875,
3584
+ "learning_rate": 1.8152054698255194e-05,
3585
+ "loss": 2.8671,
3586
+ "step": 23200
3587
+ },
3588
+ {
3589
+ "epoch": 40.364583333333336,
3590
+ "grad_norm": 3.609375,
3591
+ "learning_rate": 1.7839372250036534e-05,
3592
+ "loss": 2.8812,
3593
+ "step": 23250
3594
+ },
3595
+ {
3596
+ "epoch": 40.451388888888886,
3597
+ "grad_norm": 3.9375,
3598
+ "learning_rate": 1.7529142401821062e-05,
3599
+ "loss": 2.8657,
3600
+ "step": 23300
3601
+ },
3602
+ {
3603
+ "epoch": 40.53819444444444,
3604
+ "grad_norm": 3.875,
3605
+ "learning_rate": 1.722137441436721e-05,
3606
+ "loss": 2.8782,
3607
+ "step": 23350
3608
+ },
3609
+ {
3610
+ "epoch": 40.625,
3611
+ "grad_norm": 3.984375,
3612
+ "learning_rate": 1.6916077474943736e-05,
3613
+ "loss": 2.8685,
3614
+ "step": 23400
3615
+ },
3616
+ {
3617
+ "epoch": 40.71180555555556,
3618
+ "grad_norm": 2.90625,
3619
+ "learning_rate": 1.66132606970554e-05,
3620
+ "loss": 2.8671,
3621
+ "step": 23450
3622
+ },
3623
+ {
3624
+ "epoch": 40.798611111111114,
3625
+ "grad_norm": 3.328125,
3626
+ "learning_rate": 1.631293312017099e-05,
3627
+ "loss": 2.8723,
3628
+ "step": 23500
3629
+ },
3630
+ {
3631
+ "epoch": 40.885416666666664,
3632
+ "grad_norm": 3.171875,
3633
+ "learning_rate": 1.6015103709453482e-05,
3634
+ "loss": 2.8591,
3635
+ "step": 23550
3636
+ },
3637
+ {
3638
+ "epoch": 40.97222222222222,
3639
+ "grad_norm": 3.375,
3640
+ "learning_rate": 1.571978135549238e-05,
3641
+ "loss": 2.8635,
3642
+ "step": 23600
3643
+ },
3644
+ {
3645
+ "epoch": 41.0,
3646
+ "eval_loss": 2.8541696071624756,
3647
+ "eval_runtime": 40.7796,
3648
+ "eval_samples_per_second": 91.443,
3649
+ "eval_steps_per_second": 5.738,
3650
+ "step": 23616
3651
+ },
3652
+ {
3653
+ "epoch": 41.05902777777778,
3654
+ "grad_norm": 3.5625,
3655
+ "learning_rate": 1.5426974874038247e-05,
3656
+ "loss": 2.8627,
3657
+ "step": 23650
3658
+ },
3659
+ {
3660
+ "epoch": 41.145833333333336,
3661
+ "grad_norm": 2.96875,
3662
+ "learning_rate": 1.51366930057398e-05,
3663
+ "loss": 2.8606,
3664
+ "step": 23700
3665
+ },
3666
+ {
3667
+ "epoch": 41.232638888888886,
3668
+ "grad_norm": 3.25,
3669
+ "learning_rate": 1.4848944415882648e-05,
3670
+ "loss": 2.8608,
3671
+ "step": 23750
3672
+ },
3673
+ {
3674
+ "epoch": 41.31944444444444,
3675
+ "grad_norm": 2.9375,
3676
+ "learning_rate": 1.4563737694130885e-05,
3677
+ "loss": 2.8802,
3678
+ "step": 23800
3679
+ },
3680
+ {
3681
+ "epoch": 41.40625,
3682
+ "grad_norm": 3.21875,
3683
+ "learning_rate": 1.4281081354270564e-05,
3684
+ "loss": 2.8615,
3685
+ "step": 23850
3686
+ },
3687
+ {
3688
+ "epoch": 41.49305555555556,
3689
+ "grad_norm": 3.59375,
3690
+ "learning_rate": 1.4000983833955594e-05,
3691
+ "loss": 2.8829,
3692
+ "step": 23900
3693
+ },
3694
+ {
3695
+ "epoch": 41.579861111111114,
3696
+ "grad_norm": 2.828125,
3697
+ "learning_rate": 1.3723453494455784e-05,
3698
+ "loss": 2.8665,
3699
+ "step": 23950
3700
+ },
3701
+ {
3702
+ "epoch": 41.666666666666664,
3703
+ "grad_norm": 3.015625,
3704
+ "learning_rate": 1.3448498620407345e-05,
3705
+ "loss": 2.8761,
3706
+ "step": 24000
3707
+ },
3708
+ {
3709
+ "epoch": 41.75347222222222,
3710
+ "grad_norm": 4.03125,
3711
+ "learning_rate": 1.3176127419565564e-05,
3712
+ "loss": 2.8624,
3713
+ "step": 24050
3714
+ },
3715
+ {
3716
+ "epoch": 41.84027777777778,
3717
+ "grad_norm": 3.09375,
3718
+ "learning_rate": 1.2906348022559755e-05,
3719
+ "loss": 2.8687,
3720
+ "step": 24100
3721
+ },
3722
+ {
3723
+ "epoch": 41.927083333333336,
3724
+ "grad_norm": 3.078125,
3725
+ "learning_rate": 1.2639168482650532e-05,
3726
+ "loss": 2.8575,
3727
+ "step": 24150
3728
+ },
3729
+ {
3730
+ "epoch": 42.0,
3731
+ "eval_loss": 2.854001045227051,
3732
+ "eval_runtime": 40.3709,
3733
+ "eval_samples_per_second": 92.368,
3734
+ "eval_steps_per_second": 5.796,
3735
+ "step": 24192
3736
+ },
3737
+ {
3738
+ "epoch": 42.013888888888886,
3739
+ "grad_norm": 3.96875,
3740
+ "learning_rate": 1.2374596775489477e-05,
3741
+ "loss": 2.8656,
3742
+ "step": 24200
3743
+ },
3744
+ {
3745
+ "epoch": 42.10069444444444,
3746
+ "grad_norm": 3.4375,
3747
+ "learning_rate": 1.2112640798881058e-05,
3748
+ "loss": 2.8625,
3749
+ "step": 24250
3750
+ },
3751
+ {
3752
+ "epoch": 42.1875,
3753
+ "grad_norm": 3.53125,
3754
+ "learning_rate": 1.1853308372546756e-05,
3755
+ "loss": 2.8571,
3756
+ "step": 24300
3757
+ },
3758
+ {
3759
+ "epoch": 42.27430555555556,
3760
+ "grad_norm": 3.21875,
3761
+ "learning_rate": 1.1596607237891766e-05,
3762
+ "loss": 2.8664,
3763
+ "step": 24350
3764
+ },
3765
+ {
3766
+ "epoch": 42.361111111111114,
3767
+ "grad_norm": 3.125,
3768
+ "learning_rate": 1.1342545057773846e-05,
3769
+ "loss": 2.881,
3770
+ "step": 24400
3771
+ },
3772
+ {
3773
+ "epoch": 42.447916666666664,
3774
+ "grad_norm": 3.453125,
3775
+ "learning_rate": 1.1091129416274603e-05,
3776
+ "loss": 2.8614,
3777
+ "step": 24450
3778
+ },
3779
+ {
3780
+ "epoch": 42.53472222222222,
3781
+ "grad_norm": 3.28125,
3782
+ "learning_rate": 1.0842367818472988e-05,
3783
+ "loss": 2.8773,
3784
+ "step": 24500
3785
+ },
3786
+ {
3787
+ "epoch": 42.62152777777778,
3788
+ "grad_norm": 2.640625,
3789
+ "learning_rate": 1.0596267690221496e-05,
3790
+ "loss": 2.874,
3791
+ "step": 24550
3792
+ },
3793
+ {
3794
+ "epoch": 42.708333333333336,
3795
+ "grad_norm": 4.0625,
3796
+ "learning_rate": 1.0352836377924202e-05,
3797
+ "loss": 2.8666,
3798
+ "step": 24600
3799
+ },
3800
+ {
3801
+ "epoch": 42.795138888888886,
3802
+ "grad_norm": 3.203125,
3803
+ "learning_rate": 1.0112081148317687e-05,
3804
+ "loss": 2.8681,
3805
+ "step": 24650
3806
+ },
3807
+ {
3808
+ "epoch": 42.88194444444444,
3809
+ "grad_norm": 3.15625,
3810
+ "learning_rate": 9.874009188253974e-06,
3811
+ "loss": 2.8575,
3812
+ "step": 24700
3813
+ },
3814
+ {
3815
+ "epoch": 42.96875,
3816
+ "grad_norm": 3.046875,
3817
+ "learning_rate": 9.63862760448616e-06,
3818
+ "loss": 2.8666,
3819
+ "step": 24750
3820
+ },
3821
+ {
3822
+ "epoch": 43.0,
3823
+ "eval_loss": 2.8540420532226562,
3824
+ "eval_runtime": 42.3273,
3825
+ "eval_samples_per_second": 88.099,
3826
+ "eval_steps_per_second": 5.528,
3827
+ "step": 24768
3828
+ },
3829
+ {
3830
+ "epoch": 43.05555555555556,
3831
+ "grad_norm": 3.265625,
3832
+ "learning_rate": 9.405943423456043e-06,
3833
+ "loss": 2.8636,
3834
+ "step": 24800
3835
+ },
3836
+ {
3837
+ "epoch": 43.142361111111114,
3838
+ "grad_norm": 3.25,
3839
+ "learning_rate": 9.175963591084546e-06,
3840
+ "loss": 2.858,
3841
+ "step": 24850
3842
+ },
3843
+ {
3844
+ "epoch": 43.229166666666664,
3845
+ "grad_norm": 4.59375,
3846
+ "learning_rate": 8.948694972564343e-06,
3847
+ "loss": 2.8629,
3848
+ "step": 24900
3849
+ },
3850
+ {
3851
+ "epoch": 43.31597222222222,
3852
+ "grad_norm": 3.078125,
3853
+ "learning_rate": 8.724144352154861e-06,
3854
+ "loss": 2.8783,
3855
+ "step": 24950
3856
+ },
3857
+ {
3858
+ "epoch": 43.40277777777778,
3859
+ "grad_norm": 3.515625,
3860
+ "learning_rate": 8.502318432979806e-06,
3861
+ "loss": 2.8623,
3862
+ "step": 25000
3863
+ },
3864
+ {
3865
+ "epoch": 43.489583333333336,
3866
+ "grad_norm": 2.65625,
3867
+ "learning_rate": 8.28322383682707e-06,
3868
+ "loss": 2.8827,
3869
+ "step": 25050
3870
+ },
3871
+ {
3872
+ "epoch": 43.576388888888886,
3873
+ "grad_norm": 3.203125,
3874
+ "learning_rate": 8.066867103951082e-06,
3875
+ "loss": 2.8631,
3876
+ "step": 25100
3877
+ },
3878
+ {
3879
+ "epoch": 43.66319444444444,
3880
+ "grad_norm": 2.890625,
3881
+ "learning_rate": 7.853254692877476e-06,
3882
+ "loss": 2.8769,
3883
+ "step": 25150
3884
+ },
3885
+ {
3886
+ "epoch": 43.75,
3887
+ "grad_norm": 3.078125,
3888
+ "learning_rate": 7.642392980210423e-06,
3889
+ "loss": 2.8654,
3890
+ "step": 25200
3891
+ },
3892
+ {
3893
+ "epoch": 43.83680555555556,
3894
+ "grad_norm": 3.21875,
3895
+ "learning_rate": 7.4342882604422125e-06,
3896
+ "loss": 2.87,
3897
+ "step": 25250
3898
+ },
3899
+ {
3900
+ "epoch": 43.923611111111114,
3901
+ "grad_norm": 3.34375,
3902
+ "learning_rate": 7.228946745765364e-06,
3903
+ "loss": 2.8584,
3904
+ "step": 25300
3905
+ },
3906
+ {
3907
+ "epoch": 44.0,
3908
+ "eval_loss": 2.8539493083953857,
3909
+ "eval_runtime": 42.0373,
3910
+ "eval_samples_per_second": 88.707,
3911
+ "eval_steps_per_second": 5.566,
3912
+ "step": 25344
3913
+ },
3914
+ {
3915
+ "epoch": 44.010416666666664,
3916
+ "grad_norm": 2.90625,
3917
+ "learning_rate": 7.026374565887117e-06,
3918
+ "loss": 2.8638,
3919
+ "step": 25350
3920
+ },
3921
+ {
3922
+ "epoch": 44.09722222222222,
3923
+ "grad_norm": 2.46875,
3924
+ "learning_rate": 6.826577767846665e-06,
3925
+ "loss": 2.8638,
3926
+ "step": 25400
3927
+ },
3928
+ {
3929
+ "epoch": 44.18402777777778,
3930
+ "grad_norm": 3.078125,
3931
+ "learning_rate": 6.629562315834348e-06,
3932
+ "loss": 2.8536,
3933
+ "step": 25450
3934
+ },
3935
+ {
3936
+ "epoch": 44.270833333333336,
3937
+ "grad_norm": 3.09375,
3938
+ "learning_rate": 6.435334091013856e-06,
3939
+ "loss": 2.8646,
3940
+ "step": 25500
3941
+ }
3942
+ ],
3943
+ "logging_steps": 50,
3944
+ "max_steps": 28800,
3945
+ "num_input_tokens_seen": 0,
3946
+ "num_train_epochs": 50,
3947
+ "save_steps": 500,
3948
+ "stateful_callbacks": {
3949
+ "TrainerControl": {
3950
+ "args": {
3951
+ "should_epoch_stop": false,
3952
+ "should_evaluate": false,
3953
+ "should_log": false,
3954
+ "should_save": true,
3955
+ "should_training_stop": false
3956
+ },
3957
+ "attributes": {}
3958
+ }
3959
+ },
3960
+ "total_flos": 7.177294293290189e+18,
3961
+ "train_batch_size": 4,
3962
+ "trial_name": null,
3963
+ "trial_params": null
3964
+ }
LLM/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
README.md ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - ar
4
+ base_model:
5
+ - SparkAudio/Spark-TTS-0.5B
6
+ tags:
7
+ - speech
8
+ - arabic
9
+ - spark
10
+ - tts
11
+ - text-to-speech
12
+ license: fair-noncommercial-research-license
13
+ ---
14
+ # Spark-TTS Arabic
15
+ ## نموذج تحويل النص إلى كلام باللغة العربية
16
+
17
+ Arabic text-to-speech model fine-tuned on 300 hours of clean Arabic audio data. Delivers consistent, high-quality speech synthesis for Modern Standard Arabic with full diacritization.
18
+
19
+ ## Model Details
20
+
21
+ **Training Data:** ~300 hours of clean Arabic audio
22
+ **Language:** Modern Standard Arabic (MSA)
23
+ **Sample Rate:** 24kHz
24
+
25
+ ## Usage
26
+
27
+ ### Quick Start
28
+
29
+ see the [Colab notebook](https://colab.research.google.com/drive/1-Jxgy8BjvyWHKppdBPtz4s35Er3qDv-K?usp=sharing).
30
+ HF space : [Arabic Spark TTS Space](https://huggingface.co/spaces/IbrahimSalah/Arabic-TTS-Spark).
31
+
32
+
33
+
34
+ ```python
35
+ from transformers import AutoProcessor, AutoModel
36
+ import soundfile as sf
37
+ import torch
38
+
39
+ # Load model
40
+ model_id = "IbrahimSalah/Arabic-TTS-Spark"
41
+ device = "cuda" if torch.cuda.is_available() else "cpu"
42
+
43
+ processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
44
+ model = AutoModel.from_pretrained(model_id, trust_remote_code=True).eval().to(device)
45
+
46
+ # Prepare inputs
47
+ inputs = processor(
48
+ text="YOUR_TEXT_WITH_TASHKEEL",
49
+ prompt_speech_path="path/to/reference.wav",
50
+ prompt_text="REFERENCE_TEXT_WITH_TASHKEEL",
51
+ return_tensors="pt"
52
+ ).to(device)
53
+
54
+ # Generate
55
+ with torch.no_grad():
56
+ output_ids = model.generate(**inputs, max_new_tokens=8000, temperature=0.8)
57
+
58
+ # Decode
59
+ output = processor.decode(generated_ids=output_ids)
60
+ sf.write("output.wav", output["audio"], output["sampling_rate"])
61
+ ```
62
+
63
+ ## Key Features
64
+
65
+ - High-quality Arabic speech synthesis with natural prosody
66
+ - Efficient voice cloning from reference audio
67
+ - Advanced text chunking for long-form content
68
+ - Built-in audio post-processing (normalization, silence removal, crossfading)
69
+ - Works best with moderate text lengths
70
+ - Adjustable generation parameters (temperature, top_k, top_p)
71
+
72
+ ## Input Requirements
73
+
74
+ **Critical:** Text must include full Arabic diacritization (tashkeel). The model is trained exclusively on fully diacritized text and will not perform well on non-diacritized input.
75
+
76
+ Example of correct input:
77
+ ```
78
+ إِنَّ الْعِلْمَ نُورٌ يُقْذَفُ فِي الْقَلْبِ
79
+ ```
80
+
81
+ ### Generation Parameters
82
+
83
+ ```python
84
+ tts.generate_long_text(
85
+ text=your_text,
86
+ prompt_audio_path="reference.wav",
87
+ prompt_transcript="reference_text",
88
+ output_path="output.wav",
89
+ max_chunk_length=300, # Characters per chunk
90
+ crossfade_duration=0.08, # Crossfade duration in seconds
91
+ normalize_audio_flag=True,
92
+ remove_silence_flag=True,
93
+ temperature=0.8, # Generation randomness
94
+ top_p=0.95, # Nucleus sampling
95
+ top_k=50 # Top-k sampling
96
+ )
97
+ ```
98
+
99
+ ## Sample Output
100
+
101
+ **Text:** "إِنَّ الدَّوْلَةَ لَهَا أَعْمَارٌ طَبِيعِيَّةٌ كَمَا لِلْأَشْخَاصِ. وَأَنَّهَا تَنْتَقِلُ فِي أَطْوَارٍ مُخْتَلِفَةٍ، فَيَكُونُ الْجِيلُ الْأَوَّلُ مِنْ أَهْلِ الدَّوْلَةِ، قَدْ حَافَظُوا عَلَى الْخُشُونَةِ الْبَدَوِيَّةِ، وَالتَّوَحُّشِ، وَالشَّظَفِ، وَالْبَأْسِ، وَالِاشْتِرَاكِ فِي الْمَجْدِ. فَتَكُونُ حُدُودُهُمْ مَرْهُوبَةً، وَجَوَانِبُهُمْ مُعَزَّزَةً. ثُمَّ يَأْتِي الْجِيلُ الثَّانِي، فَيَتَحَوَّلُ حَالُهُمْ بِالْمُلْكِ وَالتَّرَفِ مِنَ الْبَدَاوَةِ إِلَى الْحَضَارَةِ، وَمِنَ الْخُشُونَةِ إِلَى التَّرَفِ. فَيَنْكَسِرُ سَوْرَةُ الْعَصَبِيَّةِ قَلِيلًا. ثُمَّ يَأْتِي الْجِيلُ الثَّالِثُ، فَيَكُونُونَ قَدْ نَسُوا عَهْدَ الْبَدَاوَةِ وَالْخُشُونَةِ، وَيَنْغَمِسُونَ فِي النَّعِيمِ وَالتَّرَفِ، وَيَصِيرُونَ عِيَالًا عَلَى الدَّوْلَةِ. فَيَسْقُطُونَ فِي الْهَرَمِ وَالزَّوَالِ، وَيَحْتَاجُونَ إِلَى مَنْ يُدَافِعُ عَنْهُمْ، فَتَبْدَأُ الدَّوْلَةُ فِي الِانْقِرَاضِ."
102
+
103
+ <audio controls src="https://cdn-uploads.huggingface.co/production/uploads/645098004f731658826cfe57/FCGgeIu1F89rvNI55aVIx.wav"></audio>
104
+ ## refrence audio
105
+
106
+ <audio controls src="https://cdn-uploads.huggingface.co/production/uploads/645098004f731658826cfe57/cA9Z77_P0Rm2-hu1eosOC.wav"></audio>
107
+
108
+ ## Further Fine-tuning
109
+
110
+ The model can be further fine-tuned for:
111
+ - Non-diacritized text (requires additional training)
112
+ - Specific voice characteristics
113
+ - Domain-specific vocabulary
114
+ - Dialectal variations
115
+
116
+ Fine-tuning infrastructure: [Spark-TTS Fine-tune](https://github.com/tuan12378/Spark-TTS-finetune)
117
+
118
+
119
+ ## License
120
+
121
+ This model is released under a **Non-Commercial License**.
122
+
123
+ - You may use this model for research, educational, and personal non-commercial purposes.
124
+ - Commercial use is strictly prohibited without explicit permission.
125
+ - If you wish to use this model for commercial purposes, please contact the model author.
126
+
127
+
128
+ ## Acknowledgments
129
+
130
+ - Base model: [Spark-TTS](https://github.com/tuan12378/Spark-TTS-finetune) by tuan12378
131
+
132
+ ## Limitations
133
+
134
+ - Requires fully diacritized Arabic text as input
135
+ - Optimized for Modern Standard Arabic (MSA), not dialectal Arabic
136
+ - Performance may vary with very long texts without proper chunking
137
+ - Voice cloning quality depends on reference audio quality and length
138
+ - Generation speed scales with text length
config.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_type": "spark-tts",
3
+ "architectures": [
4
+ "SparkTTSModel"
5
+ ],
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_spark_tts.SparkTTSConfig",
8
+ "AutoModel": "modeling_spark_tts.SparkTTSModel",
9
+ "AutoProcessor": "processing_spark_tts.SparkTTSProcessor"
10
+ },
11
+ "processor_class": "processing_spark_tts.SparkTTSProcessor",
12
+ "llm_model_name_or_path": "./LLM",
13
+ "bicodec_model_name_or_path": "./BiCodec",
14
+ "wav2vec2_model_name_or_path": "./wav2vec2-large-xlsr-53",
15
+ "sample_rate": 16000,
16
+ "highpass_cutoff_freq": 40,
17
+ "latent_hop_length": 320,
18
+ "ref_segment_duration": 6.0,
19
+ "volume_normalize": true,
20
+ "torch_dtype": "bfloat16",
21
+ "transformers_version": "4.50.3",
22
+ "_commit_hash": null,
23
+ "bicodec_config": {
24
+ "mel_params": {
25
+ "sample_rate": 16000,
26
+ "n_fft": 1024,
27
+ "win_length": 640,
28
+ "hop_length": 320,
29
+ "mel_fmin": 10,
30
+ "mel_fmax": null,
31
+ "num_mels": 128
32
+ },
33
+ "encoder_config": {
34
+ "input_channels": 1024,
35
+ "vocos_dim": 384,
36
+ "vocos_intermediate_dim": 2048,
37
+ "vocos_num_layers": 12,
38
+ "out_channels": 1024,
39
+ "sample_ratios": [1, 1]
40
+ },
41
+ "decoder_config": {
42
+ "input_channel": 1024,
43
+ "channels": 1536,
44
+ "rates": [8, 5, 4, 2],
45
+ "kernel_sizes": [16, 11, 8, 4]
46
+ },
47
+ "quantizer_config": {
48
+ "input_dim": 1024,
49
+ "codebook_size": 8192,
50
+ "codebook_dim": 8,
51
+ "commitment": 0.25,
52
+ "codebook_loss_weight": 2.0,
53
+ "decay": 0.99,
54
+ "threshold_ema_dead_code": 0.2
55
+ },
56
+ "speaker_encoder_config": {
57
+ "input_dim": 128,
58
+ "out_dim": 1024,
59
+ "latent_dim": 128,
60
+ "token_num": 32,
61
+ "fsq_levels": [4, 4, 4, 4, 4, 4],
62
+ "fsq_num_quantizers": 1
63
+ },
64
+ "prenet_config": {
65
+ "input_channels": 1024,
66
+ "vocos_dim": 384,
67
+ "vocos_intermediate_dim": 2048,
68
+ "vocos_num_layers": 12,
69
+ "out_channels": 1024,
70
+ "condition_dim": 1024,
71
+ "sample_ratios": [1, 1],
72
+ "use_tanh_at_final": false
73
+ },
74
+ "postnet_config": {
75
+ "input_channels": 1024,
76
+ "vocos_dim": 384,
77
+ "vocos_intermediate_dim": 2048,
78
+ "vocos_num_layers": 6,
79
+ "out_channels": 1024,
80
+ "use_tanh_at_final": false
81
+ }
82
+ }
83
+ }
config.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ highpass_cutoff_freq: 40
2
+ sample_rate: 16000
3
+ segment_duration: 2.4 # (s)
4
+ max_val_duration: 12 # (s)
5
+ latent_hop_length: 320
6
+ ref_segment_duration: 6
7
+ volume_normalize: true
configuration_spark_tts.py ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2025 SparkAudio & The HuggingFace Inc. team. All rights reserved.
3
+ # ... (License headers remain the same) ...
4
+ """ SparkTTS model configuration"""
5
+
6
+ from transformers.configuration_utils import PretrainedConfig
7
+ from transformers.utils import logging
8
+ from typing import List, Optional # Added typing
9
+
10
+
11
+ logger = logging.get_logger(__name__)
12
+
13
+ # --- Define Individual Sub-Component Config Classes ---
14
+
15
+ class SparkTTSMelParamsConfig(PretrainedConfig):
16
+ """Configuration for Mel Spectrogram parameters."""
17
+ model_type = "spark-tts-mel-params"
18
+ def __init__(self, sample_rate=16000, n_fft=1024, win_length=640, hop_length=320,
19
+ mel_fmin=10, mel_fmax=None, num_mels=128, **kwargs):
20
+ super().__init__(**kwargs)
21
+ self.sample_rate = sample_rate
22
+ self.n_fft = n_fft
23
+ self.win_length = win_length
24
+ self.hop_length = hop_length
25
+ self.mel_fmin = mel_fmin
26
+ self.mel_fmax = mel_fmax
27
+ self.num_mels = num_mels
28
+
29
+ class SparkTTSEncoderConfig(PretrainedConfig):
30
+ """Configuration for the BiCodec Feature Encoder."""
31
+ model_type = "spark-tts-encoder"
32
+ def __init__(self, input_channels=1024, vocos_dim=384, vocos_intermediate_dim=2048,
33
+ vocos_num_layers=12, out_channels=1024, sample_ratios=[1, 1], **kwargs):
34
+ super().__init__(**kwargs)
35
+ self.input_channels = input_channels
36
+ self.vocos_dim = vocos_dim
37
+ self.vocos_intermediate_dim = vocos_intermediate_dim
38
+ self.vocos_num_layers = vocos_num_layers
39
+ self.out_channels = out_channels
40
+ self.sample_ratios = sample_ratios
41
+
42
+ class SparkTTSDecoderConfig(PretrainedConfig):
43
+ """Configuration for the BiCodec Wave Generator (Decoder)."""
44
+ model_type = "spark-tts-decoder"
45
+ def __init__(self, input_channel=1024, channels=1536, rates=[8, 5, 4, 2],
46
+ kernel_sizes=[16, 11, 8, 4], **kwargs):
47
+ super().__init__(**kwargs)
48
+ self.input_channel = input_channel
49
+ self.channels = channels
50
+ self.rates = rates
51
+ self.kernel_sizes = kernel_sizes
52
+
53
+ class SparkTTSQuantizerConfig(PretrainedConfig):
54
+ """Configuration for the BiCodec Factorized Vector Quantizer."""
55
+ model_type = "spark-tts-quantizer"
56
+ def __init__(self, input_dim=1024, codebook_size=8192, codebook_dim=8,
57
+ commitment=0.25, codebook_loss_weight=2.0, decay=0.99,
58
+ threshold_ema_dead_code=0.2, **kwargs):
59
+ # Note: Removed use_l2_normlize as it wasn't in the original class __init__ args
60
+ # Add it back if it's actually used by the FactorizedVectorQuantize class init
61
+ super().__init__(**kwargs)
62
+ self.input_dim = input_dim
63
+ self.codebook_size = codebook_size
64
+ self.codebook_dim = codebook_dim
65
+ self.commitment = commitment
66
+ self.codebook_loss_weight = codebook_loss_weight
67
+ self.decay = decay
68
+ self.threshold_ema_dead_code = threshold_ema_dead_code
69
+
70
+ class SparkTTSSpeakerEncoderConfig(PretrainedConfig):
71
+ """Configuration for the BiCodec Speaker Encoder."""
72
+ model_type = "spark-tts-speaker-encoder"
73
+ def __init__(self, input_dim=128, out_dim=1024, latent_dim=128, token_num=32,
74
+ fsq_levels=[4, 4, 4, 4, 4, 4], fsq_num_quantizers=1, **kwargs):
75
+ super().__init__(**kwargs)
76
+ self.input_dim = input_dim
77
+ self.out_dim = out_dim
78
+ self.latent_dim = latent_dim
79
+ self.token_num = token_num
80
+ self.fsq_levels = fsq_levels
81
+ self.fsq_num_quantizers = fsq_num_quantizers
82
+
83
+ class SparkTTSPrenetConfig(PretrainedConfig):
84
+ """Configuration for the BiCodec Prenet."""
85
+ model_type = "spark-tts-prenet"
86
+ def __init__(self, input_channels=1024, vocos_dim=384, vocos_intermediate_dim=2048,
87
+ vocos_num_layers=12, out_channels=1024, condition_dim=1024,
88
+ sample_ratios=[1, 1], use_tanh_at_final=False, **kwargs):
89
+ super().__init__(**kwargs)
90
+ self.input_channels = input_channels
91
+ self.vocos_dim = vocos_dim
92
+ self.vocos_intermediate_dim = vocos_intermediate_dim
93
+ self.vocos_num_layers = vocos_num_layers
94
+ self.out_channels = out_channels
95
+ self.condition_dim = condition_dim
96
+ self.sample_ratios = sample_ratios
97
+ self.use_tanh_at_final = use_tanh_at_final
98
+
99
+ class SparkTTSPostnetConfig(PretrainedConfig):
100
+ """Configuration for the BiCodec Postnet."""
101
+ model_type = "spark-tts-postnet"
102
+ def __init__(self, input_channels=1024, vocos_dim=384, vocos_intermediate_dim=2048,
103
+ vocos_num_layers=6, out_channels=1024, use_tanh_at_final=False, **kwargs):
104
+ # Note: Removed condition_dim as it wasn't in the original config example for postnet
105
+ super().__init__(**kwargs)
106
+ self.input_channels = input_channels
107
+ self.vocos_dim = vocos_dim
108
+ self.vocos_intermediate_dim = vocos_intermediate_dim
109
+ self.vocos_num_layers = vocos_num_layers
110
+ self.out_channels = out_channels
111
+ self.use_tanh_at_final = use_tanh_at_final
112
+
113
+
114
+ # --- Define the Intermediate BiCodec Config Class ---
115
+
116
+ class SparkTTSBiCodecConfig(PretrainedConfig):
117
+ """
118
+ Intermediate configuration class for the BiCodec component within SparkTTS.
119
+ It holds instances of the individual sub-component configurations.
120
+ """
121
+ model_type = "spark-tts-bicodec"
122
+ # Map keys in the 'bicodec_config' dict to their respective classes
123
+ sub_configs = {
124
+ "mel_params": SparkTTSMelParamsConfig,
125
+ "encoder_config": SparkTTSEncoderConfig,
126
+ "decoder_config": SparkTTSDecoderConfig,
127
+ "quantizer_config": SparkTTSQuantizerConfig,
128
+ "speaker_encoder_config": SparkTTSSpeakerEncoderConfig,
129
+ "prenet_config": SparkTTSPrenetConfig,
130
+ "postnet_config": SparkTTSPostnetConfig,
131
+ }
132
+
133
+ def __init__(
134
+ self,
135
+ mel_params=None,
136
+ encoder_config=None,
137
+ decoder_config=None,
138
+ quantizer_config=None,
139
+ speaker_encoder_config=None,
140
+ prenet_config=None,
141
+ postnet_config=None,
142
+ **kwargs,
143
+ ):
144
+ super().__init__(**kwargs)
145
+
146
+ # Instantiate sub-configs from dictionaries or use defaults/provided instances
147
+ self.mel_params = self._init_sub_config(mel_params, "mel_params")
148
+ self.encoder_config = self._init_sub_config(encoder_config, "encoder_config")
149
+ self.decoder_config = self._init_sub_config(decoder_config, "decoder_config")
150
+ self.quantizer_config = self._init_sub_config(quantizer_config, "quantizer_config")
151
+ self.speaker_encoder_config = self._init_sub_config(speaker_encoder_config, "speaker_encoder_config")
152
+ self.prenet_config = self._init_sub_config(prenet_config, "prenet_config")
153
+ self.postnet_config = self._init_sub_config(postnet_config, "postnet_config")
154
+
155
+ def _init_sub_config(self, config_input, config_key):
156
+ """Helper to initialize sub-configs."""
157
+ config_cls = self.sub_configs[config_key]
158
+ if isinstance(config_input, dict):
159
+ return config_cls(**config_input)
160
+ elif config_input is None:
161
+ return config_cls() # Initialize with defaults
162
+ elif isinstance(config_input, config_cls):
163
+ return config_input # Already an instance
164
+ else:
165
+ raise TypeError(f"Invalid type for {config_key}: {type(config_input)}. Expected dict, None, or {config_cls.__name__}.")
166
+
167
+
168
+ # --- Define the Main SparkTTS Config Class ---
169
+
170
+ class SparkTTSConfig(PretrainedConfig):
171
+ r"""
172
+ Main configuration class for SparkTTSModel, including nested BiCodec configuration.
173
+ Args:
174
+ llm_model_name_or_path (`str`, *optional*, defaults to `"./LLM"`): Path/ID for LLM.
175
+ bicodec_model_name_or_path (`str`, *optional*, defaults to `"./BiCodec"`): Path/ID for BiCodec checkpoint.
176
+ wav2vec2_model_name_or_path (`str`, *optional*, defaults to `"./wav2vec2-large-xlsr-53"`): Path/ID for Wav2Vec2.
177
+ sample_rate (`int`, *optional*, defaults to 16000): Audio sample rate.
178
+ # ... (other top-level args: highpass_cutoff_freq, latent_hop_length, ref_segment_duration, volume_normalize) ...
179
+ bicodec_config (`dict`, *optional*): Dictionary to initialize `SparkTTSBiCodecConfig`.
180
+ torch_dtype (`str`, *optional*, defaults to `"auto"`): Torch dtype.
181
+ kwargs (*optional*): Dictionary of keyword arguments.
182
+ """
183
+ model_type = "spark-tts"
184
+ # Map the key in config.json to the intermediate BiCodec config class
185
+ sub_configs = {"bicodec_config": SparkTTSBiCodecConfig}
186
+ attribute_map = {"hidden_size": "d_model"} # Example
187
+
188
+ def __init__(
189
+ self,
190
+ llm_model_name_or_path="./LLM",
191
+ bicodec_model_name_or_path="./BiCodec",
192
+ wav2vec2_model_name_or_path="./wav2vec2-large-xlsr-53",
193
+ sample_rate=16000,
194
+ highpass_cutoff_freq=40,
195
+ latent_hop_length=320,
196
+ ref_segment_duration=6.0,
197
+ volume_normalize=True,
198
+ bicodec_config=None, # Expects a dictionary or None
199
+ torch_dtype="auto",
200
+ **kwargs,
201
+ ):
202
+ # --- Top-level parameters ---
203
+ self.llm_model_name_or_path = llm_model_name_or_path
204
+ self.bicodec_model_name_or_path = bicodec_model_name_or_path
205
+ self.wav2vec2_model_name_or_path = wav2vec2_model_name_or_path
206
+ self.sample_rate = sample_rate
207
+ self.highpass_cutoff_freq = highpass_cutoff_freq
208
+ self.latent_hop_length = latent_hop_length
209
+ self.ref_segment_duration = ref_segment_duration
210
+ self.volume_normalize = volume_normalize
211
+ self.torch_dtype = torch_dtype
212
+
213
+ # --- Nested BiCodec Configuration ---
214
+ # Instantiate the intermediate BiCodec config class, which will handle its own sub-configs
215
+ if isinstance(bicodec_config, dict):
216
+ self.bicodec_config = self.sub_configs["bicodec_config"](**bicodec_config)
217
+ elif bicodec_config is None:
218
+ logger.info("`bicodec_config` not provided. Initializing `SparkTTSBiCodecConfig` with its defaults.")
219
+ self.bicodec_config = self.sub_configs["bicodec_config"]()
220
+ elif isinstance(bicodec_config, self.sub_configs["bicodec_config"]):
221
+ self.bicodec_config = bicodec_config # Use existing instance
222
+ else:
223
+ raise TypeError(f"Invalid type for bicodec_config: {type(bicodec_config)}. Expected dict, None, or SparkTTSBiCodecConfig.")
224
+
225
+
226
+ # Set processor class and auto_map
227
+ kwargs["processor_class"] = kwargs.get("processor_class", "SparkTTSProcessor")
228
+ kwargs["auto_map"] = kwargs.get("auto_map", {
229
+ "AutoConfig": "configuration_spark_tts.SparkTTSConfig",
230
+ "AutoModel": "modeling_spark_tts.SparkTTSModel",
231
+ "AutoProcessor": "processing_spark_tts.SparkTTSProcessor"
232
+ })
233
+ super().__init__(**kwargs)
handler.py ADDED
@@ -0,0 +1,296 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Custom Handler for Hugging Face Inference Endpoints
3
+ Model: IbrahimSalah/Arabic-TTS-Spark
4
+ Repository: https://huggingface.co/IbrahimSalah/Arabic-TTS-Spark
5
+
6
+ This handler provides Text-to-Speech inference for Arabic with:
7
+ - Voice cloning (with reference audio)
8
+ - Controllable TTS (with gender, pitch, speed parameters)
9
+ """
10
+
11
+ import base64
12
+ import io
13
+ import logging
14
+ import os
15
+ import tempfile
16
+ from pathlib import Path
17
+ from typing import Any, Dict, Optional
18
+
19
+ import numpy as np
20
+ import soundfile as sf
21
+ import torch
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+
26
+ class EndpointHandler:
27
+ """
28
+ Hugging Face Inference Endpoints handler for Arabic-TTS-Spark.
29
+
30
+ Supports two modes:
31
+ 1. Voice Cloning: Provide reference audio to clone the voice
32
+ 2. Controllable TTS: Specify gender, pitch, and speed parameters
33
+ """
34
+
35
+ def __init__(self, path: str = ""):
36
+ """
37
+ Initialize the handler by loading the model and processor.
38
+
39
+ Args:
40
+ path: Path to the model directory (provided by HF Inference Endpoints)
41
+ """
42
+ from transformers import AutoModel, AutoProcessor
43
+
44
+ self.device = self._get_device()
45
+ logger.info(f"Initializing Arabic-TTS-Spark on device: {self.device}")
46
+
47
+ # Determine model path
48
+ model_path = path if path else "IbrahimSalah/Arabic-TTS-Spark"
49
+ logger.info(f"Loading model from: {model_path}")
50
+
51
+ # Load processor and model with trust_remote_code=True (required for custom classes)
52
+ self.processor = AutoProcessor.from_pretrained(
53
+ model_path,
54
+ trust_remote_code=True
55
+ )
56
+
57
+ self.model = AutoModel.from_pretrained(
58
+ model_path,
59
+ trust_remote_code=True,
60
+ torch_dtype=torch.bfloat16 if self.device.type == "cuda" else torch.float32
61
+ )
62
+
63
+ # Move model to device and set to eval mode
64
+ self.model = self.model.to(self.device).eval()
65
+
66
+ # Link processor to model (required for voice cloning)
67
+ self.processor.link_model(self.model)
68
+
69
+ # Store default reference audio path
70
+ self.default_reference_path = Path(model_path) / "reference.wav"
71
+ if not self.default_reference_path.exists():
72
+ # Try to find it in the resolved path
73
+ self.default_reference_path = Path(path) / "reference.wav" if path else None
74
+
75
+ logger.info("Model loaded successfully")
76
+
77
+ def _get_device(self) -> torch.device:
78
+ """Determine the best available device."""
79
+ if torch.cuda.is_available():
80
+ return torch.device("cuda")
81
+ elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
82
+ return torch.device("mps")
83
+ return torch.device("cpu")
84
+
85
+ def _decode_audio_base64(self, audio_base64: str) -> tuple:
86
+ """
87
+ Decode base64 audio to numpy array.
88
+
89
+ Args:
90
+ audio_base64: Base64 encoded audio data
91
+
92
+ Returns:
93
+ Tuple of (audio_data, sample_rate)
94
+ """
95
+ audio_bytes = base64.b64decode(audio_base64)
96
+ audio_buffer = io.BytesIO(audio_bytes)
97
+ audio_data, sample_rate = sf.read(audio_buffer)
98
+ return audio_data, sample_rate
99
+
100
+ def _encode_audio_base64(self, audio_data: np.ndarray, sample_rate: int) -> str:
101
+ """
102
+ Encode audio numpy array to base64.
103
+
104
+ Args:
105
+ audio_data: Audio waveform as numpy array
106
+ sample_rate: Sample rate of the audio
107
+
108
+ Returns:
109
+ Base64 encoded audio string
110
+ """
111
+ audio_buffer = io.BytesIO()
112
+ sf.write(audio_buffer, audio_data, sample_rate, format='WAV')
113
+ audio_buffer.seek(0)
114
+ return base64.b64encode(audio_buffer.read()).decode('utf-8')
115
+
116
+ def _validate_inputs(self, data: Dict[str, Any]) -> tuple:
117
+ """
118
+ Validate and extract inputs from request data.
119
+
120
+ Args:
121
+ data: Request data dictionary
122
+
123
+ Returns:
124
+ Tuple of (text, parameters, mode)
125
+ """
126
+ # Extract text input
127
+ text = data.get("inputs", "")
128
+ if not text:
129
+ raise ValueError("No input text provided. Use 'inputs' field.")
130
+
131
+ # Extract parameters
132
+ parameters = data.get("parameters", {})
133
+
134
+ # Determine mode
135
+ has_audio = "prompt_audio_base64" in parameters or "prompt_audio" in parameters
136
+ has_control = all(k in parameters for k in ["gender", "pitch", "speed"])
137
+
138
+ if has_audio:
139
+ mode = "voice_cloning"
140
+ elif has_control:
141
+ mode = "controllable"
142
+ else:
143
+ # Default to controllable with default parameters
144
+ mode = "controllable"
145
+ parameters.setdefault("gender", "male")
146
+ parameters.setdefault("pitch", "moderate")
147
+ parameters.setdefault("speed", "moderate")
148
+
149
+ return text, parameters, mode
150
+
151
+ def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]:
152
+ """
153
+ Process inference request.
154
+
155
+ Args:
156
+ data: Request data with the following structure:
157
+ {
158
+ "inputs": "Arabic text with diacritics",
159
+ "parameters": {
160
+ # For voice cloning:
161
+ "prompt_audio_base64": "<base64-wav>", # or "prompt_audio"
162
+ "prompt_text": "reference transcript",
163
+
164
+ # For controllable TTS:
165
+ "gender": "male" or "female",
166
+ "pitch": "very_low", "low", "moderate", "high", "very_high",
167
+ "speed": "very_low", "low", "moderate", "high", "very_high",
168
+
169
+ # Generation parameters (optional):
170
+ "temperature": 0.8,
171
+ "max_new_tokens": 3000,
172
+ "top_p": 0.95,
173
+ "top_k": 50
174
+ }
175
+ }
176
+
177
+ Returns:
178
+ Dictionary with:
179
+ {
180
+ "audio": "<base64-encoded-wav>",
181
+ "sampling_rate": 16000
182
+ }
183
+ """
184
+ try:
185
+ # Validate inputs
186
+ text, parameters, mode = self._validate_inputs(data)
187
+ logger.info(f"Processing request - Mode: {mode}, Text length: {len(text)}")
188
+
189
+ # Extract generation parameters
190
+ temperature = parameters.get("temperature", 0.8)
191
+ max_new_tokens = parameters.get("max_new_tokens", 3000)
192
+ top_p = parameters.get("top_p", 0.95)
193
+ top_k = parameters.get("top_k", 50)
194
+
195
+ # Prepare processor inputs based on mode
196
+ if mode == "voice_cloning":
197
+ # Handle voice cloning mode
198
+ audio_base64 = parameters.get("prompt_audio_base64") or parameters.get("prompt_audio")
199
+ prompt_text = parameters.get("prompt_text", "")
200
+
201
+ # Save audio to temporary file (processor expects file path)
202
+ with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp_file:
203
+ audio_data, _ = self._decode_audio_base64(audio_base64)
204
+ sf.write(tmp_file.name, audio_data, 16000)
205
+ tmp_audio_path = tmp_file.name
206
+
207
+ try:
208
+ # Process inputs for voice cloning
209
+ inputs = self.processor(
210
+ text=text,
211
+ prompt_speech_path=tmp_audio_path,
212
+ prompt_text=prompt_text if prompt_text else None,
213
+ return_tensors="pt"
214
+ )
215
+ finally:
216
+ # Clean up temporary file
217
+ os.unlink(tmp_audio_path)
218
+ else:
219
+ # Handle controllable TTS mode
220
+ gender = parameters.get("gender", "male")
221
+ pitch = parameters.get("pitch", "moderate")
222
+ speed = parameters.get("speed", "moderate")
223
+
224
+ # Validate parameter values
225
+ valid_genders = ["male", "female"]
226
+ valid_levels = ["very_low", "low", "moderate", "high", "very_high"]
227
+
228
+ if gender not in valid_genders:
229
+ raise ValueError(f"Invalid gender: {gender}. Must be one of {valid_genders}")
230
+ if pitch not in valid_levels:
231
+ raise ValueError(f"Invalid pitch: {pitch}. Must be one of {valid_levels}")
232
+ if speed not in valid_levels:
233
+ raise ValueError(f"Invalid speed: {speed}. Must be one of {valid_levels}")
234
+
235
+ # Process inputs for controllable TTS
236
+ inputs = self.processor(
237
+ text=text,
238
+ gender=gender,
239
+ pitch=pitch,
240
+ speed=speed,
241
+ return_tensors="pt"
242
+ )
243
+
244
+ # Move inputs to device
245
+ inputs = {k: v.to(self.device) if isinstance(v, torch.Tensor) else v
246
+ for k, v in inputs.items()}
247
+
248
+ # Store input length for decoding
249
+ input_ids_len = inputs["input_ids"].shape[1]
250
+
251
+ # Generate audio tokens
252
+ with torch.no_grad():
253
+ output_ids = self.model.generate(
254
+ input_ids=inputs["input_ids"],
255
+ attention_mask=inputs.get("attention_mask"),
256
+ max_new_tokens=max_new_tokens,
257
+ temperature=temperature,
258
+ top_p=top_p,
259
+ top_k=top_k,
260
+ do_sample=True,
261
+ pad_token_id=self.processor.tokenizer.pad_token_id or self.processor.tokenizer.eos_token_id,
262
+ eos_token_id=self.processor.tokenizer.eos_token_id,
263
+ )
264
+
265
+ # Decode audio
266
+ global_tokens = inputs.get("global_token_ids_prompt")
267
+ output = self.processor.decode(
268
+ generated_ids=output_ids,
269
+ global_token_ids_prompt=global_tokens,
270
+ input_ids_len=input_ids_len
271
+ )
272
+
273
+ # Get audio data
274
+ audio_data = output["audio"]
275
+ sampling_rate = output["sampling_rate"]
276
+
277
+ # Ensure audio is valid
278
+ if audio_data is None or len(audio_data) == 0:
279
+ raise RuntimeError("Model generated empty audio output")
280
+
281
+ # Encode audio to base64
282
+ audio_base64 = self._encode_audio_base64(audio_data, sampling_rate)
283
+
284
+ logger.info(f"Generated audio: {len(audio_data)} samples at {sampling_rate}Hz")
285
+
286
+ return {
287
+ "audio": audio_base64,
288
+ "sampling_rate": sampling_rate
289
+ }
290
+
291
+ except Exception as e:
292
+ logger.error(f"Inference error: {str(e)}")
293
+ return {
294
+ "error": str(e),
295
+ "error_type": type(e).__name__
296
+ }
modeling_spark_tts.py ADDED
The diff for this file is too large to render. See raw diff
 
processing_spark_tts.py ADDED
@@ -0,0 +1,889 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2025 SparkAudio & The HuggingFace Inc. team.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """
16
+ Processor class for SparkTTS. Combines text tokenization and audio feature extraction/processing.
17
+ """
18
+
19
+ import os # Needed for save_pretrained
20
+ import re # For decoding
21
+ import torch
22
+ import numpy as np
23
+ import soundfile as sf # For audio loading
24
+ import soxr # For resampling
25
+
26
+ from pathlib import Path
27
+ from typing import Optional, Union, List, Dict, Tuple, Any
28
+
29
+ from transformers.processing_utils import ProcessorMixin
30
+ from transformers.tokenization_utils_base import BatchEncoding # Return type hint
31
+ from transformers.feature_extraction_utils import BatchFeature # Return type hint
32
+ from transformers.models.auto.tokenization_auto import AutoTokenizer
33
+ from transformers.models.wav2vec2.feature_extraction_wav2vec2 import Wav2Vec2FeatureExtractor
34
+ from transformers.utils import logging, PushToHubMixin # Added PushToHubMixin
35
+ from numpy.lib.stride_tricks import sliding_window_view
36
+ import soxr
37
+ import soundfile
38
+ import random
39
+
40
+ # Import custom config if needed for defaults
41
+ from .configuration_spark_tts import SparkTTSConfig
42
+
43
+ logger = logging.get_logger(__name__)
44
+
45
+
46
+ # =============================================================================
47
+ # >> START: PASTE CODE FROM sparktts/utils/* HERE <<
48
+ # =============================================================================
49
+ # IMPORTANT: Utility functions needed for processing (audio loading, token parsing)
50
+ # must be defined or imported here.
51
+
52
+ # --- Paste sparktts/utils/audio.py content here ---
53
+
54
+ def audio_volume_normalize(audio: np.ndarray, coeff: float = 0.2) -> np.ndarray:
55
+ """
56
+ Normalize the volume of an audio signal.
57
+
58
+ Parameters:
59
+ audio (numpy array): Input audio signal array.
60
+ coeff (float): Target coefficient for normalization, default is 0.2.
61
+
62
+ Returns:
63
+ numpy array: The volume-normalized audio signal.
64
+ """
65
+ # Sort the absolute values of the audio signal
66
+ temp = np.sort(np.abs(audio))
67
+
68
+ # If the maximum value is less than 0.1, scale the array to have a maximum of 0.1
69
+ if temp[-1] < 0.1:
70
+ scaling_factor = max(
71
+ temp[-1], 1e-3
72
+ ) # Prevent division by zero with a small constant
73
+ audio = audio / scaling_factor * 0.1
74
+
75
+ # Filter out values less than 0.01 from temp
76
+ temp = temp[temp > 0.01]
77
+ L = temp.shape[0] # Length of the filtered array
78
+
79
+ # If there are fewer than or equal to 10 significant values, return the audio without further processing
80
+ if L <= 10:
81
+ return audio
82
+
83
+ # Compute the average of the top 10% to 1% of values in temp
84
+ volume = np.mean(temp[int(0.9 * L) : int(0.99 * L)])
85
+
86
+ # Normalize the audio to the target coefficient level, clamping the scale factor between 0.1 and 10
87
+ audio = audio * np.clip(coeff / volume, a_min=0.1, a_max=10)
88
+
89
+ # Ensure the maximum absolute value in the audio does not exceed 1
90
+ max_value = np.max(np.abs(audio))
91
+ if max_value > 1:
92
+ audio = audio / max_value
93
+
94
+ return audio
95
+
96
+
97
+ def load_audio(
98
+ adfile: Path,
99
+ sampling_rate: int = None,
100
+ length: int = None,
101
+ volume_normalize: bool = False,
102
+ segment_duration: int = None,
103
+ ) -> np.ndarray:
104
+ r"""Load audio file with target sampling rate and lsength
105
+
106
+ Args:
107
+ adfile (Path): path to audio file.
108
+ sampling_rate (int, optional): target sampling rate. Defaults to None.
109
+ length (int, optional): target audio length. Defaults to None.
110
+ volume_normalize (bool, optional): whether perform volume normalization. Defaults to False.
111
+ segment_duration (int): random select a segment with duration of {segment_duration}s.
112
+ Defualt to None which means the whole audio will be used.
113
+
114
+ Returns:
115
+ audio (np.ndarray): audio
116
+ """
117
+
118
+ audio, sr = soundfile.read(adfile)
119
+ if len(audio.shape) > 1:
120
+ audio = audio[:, 0]
121
+
122
+ if sampling_rate is not None and sr != sampling_rate:
123
+ audio = soxr.resample(audio, sr, sampling_rate, quality="VHQ")
124
+ sr = sampling_rate
125
+
126
+ if segment_duration is not None:
127
+ seg_length = int(sr * segment_duration)
128
+ audio = random_select_audio_segment(audio, seg_length)
129
+
130
+ # Audio volume normalize
131
+ if volume_normalize:
132
+ audio = audio_volume_normalize(audio)
133
+ # check the audio length
134
+ if length is not None:
135
+ assert abs(audio.shape[0] - length) < 1000
136
+ if audio.shape[0] > length:
137
+ audio = audio[:length]
138
+ else:
139
+ audio = np.pad(audio, (0, int(length - audio.shape[0])))
140
+ return audio
141
+
142
+
143
+ def random_select_audio_segment(audio: np.ndarray, length: int) -> np.ndarray:
144
+ """get an audio segment given the length
145
+
146
+ Args:
147
+ audio (np.ndarray):
148
+ length (int): audio length = sampling_rate * duration
149
+ """
150
+ if audio.shape[0] < length:
151
+ audio = np.pad(audio, (0, int(length - audio.shape[0])))
152
+ start_index = random.randint(0, audio.shape[0] - length)
153
+ end_index = int(start_index + length)
154
+
155
+ return audio[start_index:end_index]
156
+
157
+ def get_ref_clip(wav: np.ndarray, config) -> np.ndarray: # Needs access to config attributes
158
+ """Get reference audio clip for speaker embedding."""
159
+ # Make sure config has sample_rate, ref_segment_duration, latent_hop_length
160
+ if not all(hasattr(config, attr) for attr in ['sample_rate', 'ref_segment_duration', 'latent_hop_length']):
161
+ raise AttributeError("Config object missing required attributes for get_ref_clip")
162
+ ref_segment_length = (
163
+ int(config.sample_rate * config.ref_segment_duration)
164
+ // config.latent_hop_length
165
+ * config.latent_hop_length
166
+ )
167
+ wav_length = len(wav)
168
+ if ref_segment_length > wav_length:
169
+ wav = np.tile(wav, ref_segment_length // wav_length + 1)
170
+ return wav[:ref_segment_length]
171
+
172
+
173
+ # --- Paste sparktts/utils/token_parser.py content here ---
174
+
175
+ TASK_TOKEN_MAP = {
176
+ "vc": "<|task_vc|>",
177
+ "tts": "<|task_tts|>",
178
+ "asr": "<|task_asr|>",
179
+ "s2s": "<|task_s2s|>",
180
+ "t2s": "<|task_t2s|>",
181
+ "understand": "<|task_understand|>",
182
+ "caption": "<|task_cap|>",
183
+ "controllable_tts": "<|task_controllable_tts|>",
184
+ "prompt_tts": "<|task_prompt_tts|>",
185
+ "speech_edit": "<|task_edit|>",
186
+ }
187
+
188
+ LEVELS_MAP = {
189
+ "very_low": 0,
190
+ "low": 1,
191
+ "moderate": 2,
192
+ "high": 3,
193
+ "very_high": 4,
194
+ }
195
+
196
+ LEVELS_MAP_UI = {
197
+ 1: 'very_low',
198
+ 2: 'low',
199
+ 3: 'moderate',
200
+ 4: 'high',
201
+ 5: 'very_high'
202
+ }
203
+
204
+ GENDER_MAP = {
205
+ "female": 0,
206
+ "male": 1,
207
+ }
208
+
209
+ AGE_MAP = {"Child": 0, "Teenager": 1, "Youth-Adult": 2, "Middle-aged": 3, "Elderly": 4}
210
+
211
+ EMO_MAP = {
212
+ "UNKNOWN": 0,
213
+ "NEUTRAL": 1,
214
+ "ANGRY": 2,
215
+ "HAPPY": 3,
216
+ "SAD": 4,
217
+ "FEARFUL": 5,
218
+ "DISGUSTED": 6,
219
+ "SURPRISED": 7,
220
+ "SARCASTIC": 8,
221
+ "EXCITED": 9,
222
+ "SLEEPY": 10,
223
+ "CONFUSED": 11,
224
+ "EMPHASIS": 12,
225
+ "LAUGHING": 13,
226
+ "SINGING": 14,
227
+ "WORRIED": 15,
228
+ "WHISPER": 16,
229
+ "ANXIOUS": 17,
230
+ "NO-AGREEMENT": 18,
231
+ "APOLOGETIC": 19,
232
+ "CONCERNED": 20,
233
+ "ENUNCIATED": 21,
234
+ "ASSERTIVE": 22,
235
+ "ENCOURAGING": 23,
236
+ "CONTEMPT": 24,
237
+ }
238
+
239
+
240
+ class TokenParser:
241
+ """Turn label to special token"""
242
+
243
+ def __init__(self):
244
+ pass
245
+
246
+ """Parse the attributes of a person."""
247
+
248
+ def __init__(self):
249
+ pass
250
+
251
+ @staticmethod
252
+ def age(age: str) -> str:
253
+ """Turn age token."""
254
+ age_id = AGE_MAP[age]
255
+ return f"<|age_{age_id}|>"
256
+
257
+ @staticmethod
258
+ def gender(gender: str) -> str:
259
+ """Turn gender token."""
260
+ gender_id = GENDER_MAP[gender]
261
+ return f"<|gender_{gender_id}|>"
262
+
263
+ @staticmethod
264
+ def mel_value(mel: int):
265
+ """Turn special token of mel scale pitch."""
266
+ mel = max(0, int(mel))
267
+ mel = min(1000, int(mel))
268
+ return f"<|pitch_value_{mel}|>"
269
+
270
+ @staticmethod
271
+ def mel_level(level: str):
272
+ """Turn special token of mel level."""
273
+ level_tag = LEVELS_MAP[level]
274
+ return f"<|pitch_label_{level_tag}|>"
275
+
276
+ @staticmethod
277
+ def pitch_var_value(pitch_std: int):
278
+ """Turn special token of pitch_std value."""
279
+ assert isinstance(pitch_std, int)
280
+ pitch_std = max(0, int(pitch_std))
281
+ pitch_std = min(10, int(pitch_std))
282
+ return f"<|pitch_var_value_{pitch_std}|>"
283
+
284
+ @staticmethod
285
+ def pitch_var_level(level: str):
286
+ """Turn special token of pitch std level."""
287
+ level_tag = LEVELS_MAP[level]
288
+ return f"<|pitch_var_label_{level_tag}|>"
289
+
290
+ @staticmethod
291
+ def loudness_value(loudness: int):
292
+ """Turn special toak of loudness value [0, 30]"""
293
+ assert loudness >= 0
294
+ loudness = max(0, int(loudness))
295
+ loudness = min(30, int(loudness))
296
+ return f"<|loudness_value_{loudness}|>"
297
+
298
+ @staticmethod
299
+ def loudness_level(level: str):
300
+ """Turn special token of loudness level."""
301
+ level_tag = LEVELS_MAP[level]
302
+ return f"<|loudness_label_{level_tag}|>"
303
+
304
+ @staticmethod
305
+ def speed_value(speed: int):
306
+ """Turn special token of speed value."""
307
+ speed = max(0, int(speed))
308
+ speed = min(10, int(speed))
309
+ return f"<|speed_value_{speed}|>"
310
+
311
+ @staticmethod
312
+ def speed_level(level: str):
313
+ """Turn special token of speed level."""
314
+ level_tag = LEVELS_MAP[level]
315
+ return f"<|speed_label_{level_tag}|>"
316
+
317
+ @staticmethod
318
+ def task(task: str) -> str:
319
+ """Turn special token of task."""
320
+ assert task in TASK_TOKEN_MAP.keys()
321
+
322
+ return TASK_TOKEN_MAP[task]
323
+
324
+ @staticmethod
325
+ def emotion(emotion: str):
326
+ emo_id = EMO_MAP[emotion]
327
+
328
+ return f"<|emotion_{emo_id}|>"
329
+
330
+ # =============================================================================
331
+ # >> END: PASTE CODE FROM sparktts/utils/* HERE <<
332
+ # =============================================================================
333
+
334
+
335
+ class SparkTTSProcessor(ProcessorMixin, PushToHubMixin): # Added PushToHubMixin
336
+ r"""
337
+ Constructs a SparkTTS processor which wraps a text tokenizer and relevant audio processing logic.
338
+
339
+ Args:
340
+ tokenizer ([`PreTrainedTokenizer`]):
341
+ An instance of [`PreTrainedTokenizer`]. This handles the text tokenization for the LLM.
342
+ feature_extractor ([`Wav2Vec2FeatureExtractor`]):
343
+ An instance of [`Wav2Vec2FeatureExtractor`]. Although Wav2Vec2 features are extracted
344
+ within the model's `tokenize_audio`, the extractor's configuration (like sampling rate)
345
+ is useful, and it aligns with the ProcessorMixin pattern.
346
+ config ([`SparkTTSConfig`], *optional*):
347
+ An instance of [`SparkTTSConfig`] to access configuration parameters like sample rate.
348
+ """
349
+ attributes = ["tokenizer", "feature_extractor"]
350
+ tokenizer_class = "AutoTokenizer"
351
+ feature_extractor_class = "Wav2Vec2FeatureExtractor" # Keep for consistency
352
+
353
+ def __init__(self, tokenizer, feature_extractor, config: Optional[SparkTTSConfig] = None, **kwargs):
354
+ super().__init__(tokenizer=tokenizer, feature_extractor=feature_extractor, **kwargs)
355
+ self.model = None
356
+ self.config = config
357
+ # Set sampling rate
358
+ if config and hasattr(config, 'sample_rate'):
359
+ self.sampling_rate = config.sample_rate
360
+ elif feature_extractor and hasattr(feature_extractor, 'sampling_rate'):
361
+ self.sampling_rate = feature_extractor.sampling_rate
362
+ else:
363
+ self.sampling_rate = 16000
364
+ logger.warning(f"Could not determine sampling rate. Defaulting to {self.sampling_rate} Hz.")
365
+
366
+ # # Ensure tokenizer pad token
367
+ # if self.tokenizer.pad_token is None:
368
+ # if self.tokenizer.eos_token is not None:
369
+ # logger.warning("Tokenizer does not have a pad token. Setting pad_token to eos_token.")
370
+ # self.tokenizer.pad_token = self.tokenizer.eos_token
371
+ # else:
372
+ # logger.warning("Tokenizer lacks pad and eos token. Adding default pad token '<|pad|>'.")
373
+ # self.tokenizer.add_special_tokens({'pad_token': '<|pad|>'})
374
+
375
+ def link_model(self, model):
376
+ """Links the processor to a SparkTTSModel instance for audio processing calls."""
377
+ if not hasattr(model, 'tokenize_audio') or not hasattr(model, 'detokenize_audio'):
378
+ raise TypeError("The provided model instance does not have the required 'tokenize_audio' and 'detokenize_audio' methods.")
379
+ if not hasattr(model, 'config'):
380
+ logger.warning("Linked model does not have a 'config' attribute. Some processor functionalities might rely on it.")
381
+
382
+ self.model = model
383
+ logger.info("SparkTTSModel successfully linked to the processor.")
384
+ # Update sampling rate based on linked model's config if available
385
+ if hasattr(model, 'config') and hasattr(model.config, 'sample_rate'):
386
+ if self.sampling_rate != model.config.sample_rate:
387
+ logger.info(f"Updating processor sampling rate from {self.sampling_rate} to {model.config.sample_rate} based on linked model config.")
388
+ self.sampling_rate = model.config.sample_rate
389
+ # Also update feature extractor sampling rate if it differs
390
+ if hasattr(self, 'feature_extractor') and self.feature_extractor.sampling_rate != model.config.sample_rate:
391
+ logger.info(f"Updating feature_extractor sampling rate from {self.feature_extractor.sampling_rate} to {model.config.sample_rate}.")
392
+ self.feature_extractor.sampling_rate = model.config.sample_rate
393
+
394
+
395
+ def __call__(
396
+ self,
397
+ text: str,
398
+ prompt_speech_path: Optional[Union[str, Path]] = None,
399
+ prompt_text: Optional[str] = None,
400
+ gender: Optional[str] = None,
401
+ pitch: Optional[str] = None,
402
+ speed: Optional[str] = None,
403
+ return_tensors: Optional[str] = "pt",
404
+ **kwargs, # Allow passing other args like padding, truncation to tokenizer
405
+ ) -> BatchEncoding:
406
+ """
407
+ Processes the input text and optional prompt audio/control parameters into a format suitable for [`SparkTTSModel`].
408
+
409
+ Args:
410
+ text (`str`):
411
+ The main text to be synthesized.
412
+ prompt_speech_path (`str` or `Path`, *optional*):
413
+ Path to the prompt audio file for voice cloning. Required if `gender` is not set.
414
+ prompt_text (`str`, *optional*):
415
+ Transcript of the prompt audio. Used only in voice cloning mode.
416
+ gender (`str`, *optional*):
417
+ Target gender ("male" or "female") for controllable synthesis. If set, enables control mode.
418
+ pitch (`str`, *optional*):
419
+ Target pitch level ("very_low", "low", "moderate", "high", "very_high") for control mode. Required if `gender` is set.
420
+ speed (`str`, *optional*):
421
+ Target speed level ("very_low", "low", "moderate", "high", "very_high") for control mode. Required if `gender` is set.
422
+ return_tensors (`str`, *optional*, defaults to `"pt"`):
423
+ If set, will return tensors instead of list of python integers. Only "pt" (PyTorch) is supported currently.
424
+ **kwargs:
425
+ Additional arguments passed to the underlying tokenizer's `__call__` method.
426
+
427
+ Returns:
428
+ [`BatchEncoding`]: A dictionary containing the `input_ids` and `attention_mask` for the LLM.
429
+ In voice cloning mode, it also includes `global_token_ids_prompt` (torch.Tensor) representing the
430
+ global tokens extracted from the prompt audio.
431
+ """
432
+
433
+ global_token_ids_prompt = None # Initialize
434
+
435
+ # Determine mode: Control TTS or Voice Cloning (Prompt TTS)
436
+ is_control_mode = gender is not None
437
+ is_cloning_mode = prompt_speech_path is not None and not is_control_mode
438
+
439
+ if is_control_mode:
440
+ # --- Controllable TTS Prompt Construction ---
441
+ if not all([pitch, speed]):
442
+ raise ValueError("For controllable TTS, 'gender', 'pitch', and 'speed' must all be provided.")
443
+ if prompt_speech_path is not None:
444
+ logger.warning("`prompt_speech_path` provided but ignored because `gender` is set (controllable TTS mode).")
445
+
446
+ if not all(k in GENDER_MAP for k in [gender]): # Basic check
447
+ raise ValueError(f"Invalid gender provided: {gender}. Must be one of {list(GENDER_MAP.keys())}")
448
+ if not all(k in LEVELS_MAP for k in [pitch, speed]): # Basic check
449
+ raise ValueError(f"Invalid pitch or speed level provided. Must be one of {list(LEVELS_MAP.keys())}")
450
+
451
+ gender_id = GENDER_MAP[gender]
452
+ pitch_level_id = LEVELS_MAP[pitch]
453
+ speed_level_id = LEVELS_MAP[speed]
454
+
455
+ pitch_label_tokens = f"<|pitch_label_{pitch_level_id}|>"
456
+ speed_label_tokens = f"<|speed_label_{speed_level_id}|>"
457
+ gender_tokens = f"<|gender_{gender_id}|>"
458
+
459
+ attribute_tokens = "".join([gender_tokens, pitch_label_tokens, speed_label_tokens])
460
+
461
+ prompt_list = [
462
+ TASK_TOKEN_MAP["controllable_tts"],
463
+ "<|start_content|>",
464
+ text,
465
+ "<|end_content|>",
466
+ "<|start_style_label|>",
467
+ attribute_tokens,
468
+ "<|end_style_label|>",
469
+ ]
470
+ prompt_string = "".join(prompt_list)
471
+
472
+ elif is_cloning_mode:
473
+ # --- Voice Cloning Prompt Construction ---
474
+ if self.model is None:
475
+ raise RuntimeError("Processor must be linked to a SparkTTSModel instance via `processor.link_model(model)` before performing voice cloning.")
476
+ prompt_speech_path = Path(prompt_speech_path) # Ensure it's a Path object
477
+ if not prompt_speech_path.exists():
478
+ raise FileNotFoundError(f"Prompt audio file not found: {prompt_speech_path}")
479
+
480
+ # Load and process prompt audio
481
+ try:
482
+ model_config = self.model.config if self.model and hasattr(self.model, 'config') else self.config
483
+ if model_config is None:
484
+ raise ValueError("Configuration not available in processor or linked model.")
485
+
486
+ # Load main wav
487
+ wav = load_audio(
488
+ prompt_speech_path,
489
+ sampling_rate=self.sampling_rate,
490
+ volume_normalize=getattr(model_config, 'volume_normalize', True), # Use getattr for safety
491
+ )
492
+ # Get reference clip
493
+ wav_ref_np = get_ref_clip(wav, model_config) # Pass config object
494
+ wav_ref = torch.from_numpy(wav_ref_np).unsqueeze(0).float()
495
+ wav_tensor = torch.from_numpy(wav).unsqueeze(0).float()
496
+
497
+ # Tokenize using the linked model's method
498
+ # Assuming tokenize_audio returns tensors with batch dim 1: [1, N_global], [1, N_semantic]
499
+ global_tokens_tensor, semantic_tokens_tensor = self.model.tokenize_audio(wav_tensor, wav_ref)
500
+
501
+ # Store the global tokens tensor (with batch dim) for the output dict
502
+ global_token_ids_prompt = global_tokens_tensor # Keep batch dim [1, N_global]
503
+
504
+ # Convert tensors to lists of ints for string formatting
505
+ global_token_list = global_tokens_tensor.squeeze().tolist() # Remove batch dim -> list
506
+ semantic_token_list = semantic_tokens_tensor.squeeze().tolist() # Remove batch dim -> list
507
+
508
+ except Exception as e:
509
+ logger.error(f"Error processing prompt audio {prompt_speech_path}: {e}")
510
+ import traceback
511
+ traceback.print_exc()
512
+ raise
513
+
514
+ # ==============================================================
515
+ # CORRECTED TOKEN STRING FORMATTING
516
+ # ==============================================================
517
+ # Create individual token strings for each ID
518
+ global_tokens_str = "".join([f"<|bicodec_global_{gid}|>" for gid in global_token_list])
519
+ semantic_tokens_str = "".join([f"<|bicodec_semantic_{sid}|>" for sid in semantic_token_list])
520
+ # ==============================================================
521
+
522
+ # Construct prompt list based on presence of prompt_text
523
+ if prompt_text is not None and prompt_text.strip(): # Check if prompt_text is meaningful
524
+ logger.info("Using prompt text in voice cloning prompt.")
525
+ prompt_list = [
526
+ TASK_TOKEN_MAP["tts"], # Or maybe TASK_TOKEN_MAP["prompt_tts"]? Check original logic. Assuming "tts".
527
+ "<|start_content|>",
528
+ prompt_text, # Transcript first
529
+ text, # Then target text
530
+ "<|end_content|>",
531
+ "<|start_global_token|>",
532
+ global_tokens_str,
533
+ "<|end_global_token|>",
534
+ "<|start_semantic_token|>",
535
+ semantic_tokens_str,
536
+ # "<|end_semantic_token|>", # Original code didn't have this marker here
537
+ ]
538
+ else:
539
+ # Simpler prompt without semantic tokens if no transcript provided
540
+ logger.info("No prompt text provided, using text-only voice cloning prompt.")
541
+ prompt_list = [
542
+ TASK_TOKEN_MAP["tts"], # Or maybe TASK_TOKEN_MAP["prompt_tts"]?
543
+ "<|start_content|>",
544
+ text, # Only target text
545
+ "<|end_content|>",
546
+ "<|start_global_token|>",
547
+ global_tokens_str,
548
+ "<|end_global_token|>",
549
+ ]
550
+ prompt_string = "".join(prompt_list)
551
+ logger.debug(f"Generated prompt string (cloning): {prompt_string[:200]}...") # Log start of prompt
552
+
553
+ else:
554
+ raise ValueError("Invalid input combination. Either provide `prompt_speech_path` for cloning or (`gender`, `pitch`, `speed`) for control.")
555
+
556
+ # --- Tokenize the final prompt string ---
557
+ # print(f"Tokenizing prompt: {prompt_string}")
558
+ inputs = self.tokenizer(
559
+ prompt_string,
560
+ return_tensors=return_tensors,
561
+ padding=kwargs.get("padding", False), # Often False for generation prompts unless batching > 1
562
+ truncation=kwargs.get("truncation", True),
563
+ max_length=kwargs.get("max_length", self.tokenizer.model_max_length),
564
+ add_special_tokens=kwargs.get("add_special_tokens", True), # Usually True unless handled manually
565
+ return_attention_mask=kwargs.get("return_attention_mask", True), # Need attention mask
566
+ **{k: v for k, v in kwargs.items() if k not in ["padding", "truncation", "max_length", "add_special_tokens", "return_attention_mask"]}
567
+ )
568
+ logger.debug(f"Tokenized input_ids shape: {inputs['input_ids'].shape}")
569
+
570
+
571
+ # Add the prompt's global tokens (as tensor with batch dim) to the output if in cloning mode
572
+ if is_cloning_mode and global_token_ids_prompt is not None:
573
+ if return_tensors == "pt":
574
+ inputs["global_token_ids_prompt"] = global_token_ids_prompt # Already has batch dim [1, N_global]
575
+ else:
576
+ # Handle non-tensor return if necessary
577
+ inputs["global_token_ids_prompt"] = global_token_ids_prompt.tolist()
578
+
579
+ return inputs
580
+
581
+
582
+ def decode(
583
+ self,
584
+ generated_ids: torch.Tensor,
585
+ global_token_ids_prompt: Optional[torch.Tensor] = None,
586
+ input_ids_len: Optional[int] = None,
587
+ skip_special_tokens: bool = True,
588
+ ) -> Dict[str, Any]:
589
+ """
590
+ Decodes the generated token IDs from [`SparkTTSModel`] into an audio waveform.
591
+
592
+ Args:
593
+ generated_ids (`torch.Tensor`):
594
+ Tensor of token IDs generated by `model.generate()`, including the input prompt part. Shape [B, seq_len].
595
+ global_token_ids_prompt (`torch.Tensor`, *optional*):
596
+ The global tokens extracted from the prompt audio during the `__call__` step (for voice cloning).
597
+ Shape [B, N_global]. Required if the generation was for voice cloning.
598
+ input_ids_len (`int`, *optional*):
599
+ The length of the original input prompt `input_ids` fed to `model.generate()`. Required to
600
+ correctly isolate the newly generated tokens.
601
+ skip_special_tokens (`bool`, *optional*, defaults to `True`):
602
+ Whether to skip special tokens during the text decoding step (used to extract audio tokens).
603
+
604
+ Returns:
605
+ Dict[str, Any]: A dictionary containing:
606
+ - "audio": The decoded audio waveform as a NumPy array. Shape [T_audio] (if B=1) or [B, T_audio].
607
+ - "sampling_rate": The sampling rate of the audio.
608
+ """
609
+ if self.model is None:
610
+ raise RuntimeError("Processor must be linked to a SparkTTSModel instance via `processor.link_model(model)` before decoding.")
611
+ if input_ids_len is None:
612
+ raise ValueError("`input_ids_len` (length of the prompt input_ids) must be provided for decoding.")
613
+
614
+ # --- Isolate generated part and decode text ---
615
+ # Assumes generated_ids has shape [B, full_seq_len]
616
+ # Handle case where generated sequence is shorter than prompt (shouldn't happen with max_new_tokens > 0)
617
+ if generated_ids.shape[1] < input_ids_len:
618
+ logger.warning(f"Generated sequence length ({generated_ids.shape[1]}) is shorter than input prompt length ({input_ids_len}). Decoding might be incorrect.")
619
+ output_only_ids = generated_ids[:, input_ids_len:] # Will be empty if equal
620
+ else:
621
+ output_only_ids = generated_ids[:, input_ids_len:]
622
+
623
+
624
+ # Decode the generated part to find audio tokens
625
+ # Need to handle batch decoding if B > 1
626
+ # print("decode token", self.tokenizer.batch_decode(output_only_ids, skip_special_tokens=False))
627
+ decoded_texts = self.tokenizer.batch_decode(output_only_ids, skip_special_tokens=skip_special_tokens)
628
+
629
+ # --- Extract Audio Tokens ---
630
+ # Handle batch processing correctly
631
+ batch_size = generated_ids.shape[0]
632
+ all_semantic_ids = []
633
+ all_global_tokens = []
634
+ successful_indices = [] # Keep track of which batch items were successful
635
+
636
+ for i in range(batch_size):
637
+ decoded_text = decoded_texts[i]
638
+ current_semantic_ids = None
639
+ current_global_tokens = None
640
+
641
+ # Extract semantic tokens
642
+ try:
643
+ pred_semantic_indices = [int(token) for token in re.findall(r"bicodec_semantic_(\d+)", decoded_text)]
644
+ if not pred_semantic_indices:
645
+ logger.warning(f"Batch item {i}: No semantic tokens found in decoded text: '{decoded_text[:200]}...'")
646
+ continue # Skip this item
647
+
648
+ current_semantic_ids = torch.tensor(pred_semantic_indices).long() # Shape [N_semantic]
649
+ except Exception as e:
650
+ logger.error(f"Batch item {i}: Error parsing semantic tokens from: '{decoded_text[:200]}...'. Error: {e}")
651
+ continue # Skip this item
652
+
653
+ # Determine global tokens
654
+ if global_token_ids_prompt is not None:
655
+ # Cloning mode: Use the provided prompt global tokens for this batch item
656
+ if global_token_ids_prompt.shape[0] != batch_size:
657
+ raise ValueError(f"Batch size mismatch: generated_ids has {batch_size}, but global_token_ids_prompt has {global_token_ids_prompt.shape[0]}.")
658
+ current_global_tokens = global_token_ids_prompt[i] # Shape [N_global]
659
+ else:
660
+ # Control mode: Extract global tokens from the generated text
661
+ try:
662
+ pred_global_indices = [int(token) for token in re.findall(r"bicodec_global_(\d+)", decoded_text)]
663
+ if not pred_global_indices:
664
+ logger.warning(f"Batch item {i}: No global tokens found in decoded text for control mode: '{decoded_text[:200]}...'")
665
+ continue # Skip this item
666
+
667
+ current_global_tokens = torch.tensor(pred_global_indices).long() # Shape [N_global]
668
+
669
+ except Exception as e:
670
+ logger.error(f"Batch item {i}: Error parsing global tokens from: '{decoded_text[:200]}...'. Error: {e}")
671
+ continue # Skip this item
672
+
673
+ # If both tokens extracted successfully
674
+ all_semantic_ids.append(current_semantic_ids)
675
+ all_global_tokens.append(current_global_tokens)
676
+ successful_indices.append(i)
677
+
678
+ if not successful_indices:
679
+ logger.error("Failed to extract audio tokens for any item in the batch.")
680
+ return {"audio": np.array([], dtype=np.float32), "sampling_rate": self.sampling_rate}
681
+
682
+ # Pad sequences to the max length within the successful batch items for batch detokenization
683
+ # Note: BiCodec might not support batching if sequences have different lengths. Check its implementation.
684
+ # Assuming BiCodec *can* handle batches if padded (or if lengths are naturally equal).
685
+ # This padding might be unnecessary if BiCodec handles variable lengths or if B=1 anyway.
686
+ # For now, let's assume B=1 was handled correctly and skip complex padding.
687
+ if batch_size > 1 and len(successful_indices) < batch_size:
688
+ logger.warning(f"Only successfully decoded {len(successful_indices)} out of {batch_size} batch items.")
689
+ # Further processing might need to handle only the successful items.
690
+
691
+ # Let's proceed assuming B=1 or BiCodec handles batches appropriately.
692
+ # Stack the successful tokens.
693
+ try:
694
+ # Need to ensure tensors have the same length before stacking if BiCodec requires it.
695
+ # If BiCodec handles variable length, stacking might not be needed, just loop and call detokenize.
696
+ # Let's assume B=1 for simplicity of the example, matching original code's likely behavior.
697
+ if len(successful_indices) != 1:
698
+ raise NotImplementedError("Batch decoding (B > 1) requires verification of BiCodec's batch handling and potentially padding.")
699
+
700
+ final_semantic_ids = all_semantic_ids[0].unsqueeze(0) # Add batch dim [1, N_semantic]
701
+ final_global_tokens = all_global_tokens[0].unsqueeze(0) # Add batch dim [1, N_global]
702
+
703
+ except IndexError: # Should not happen if successful_indices is not empty
704
+ logger.error("Internal error during token batch preparation.")
705
+ return {"audio": np.array([], dtype=np.float32), "sampling_rate": self.sampling_rate}
706
+
707
+
708
+ # --- Detokenize Audio ---
709
+ try:
710
+ # Call the linked model's detokenize method
711
+ # print(f"DEBUG: Detokenizing audio with global tokens {final_global_tokens.shape}, semantic tokens {final_semantic_ids.shape}")
712
+ output_wav = self.model.detokenize_audio(final_global_tokens, final_semantic_ids)
713
+ # detokenize_audio now returns numpy array float32 in [-1, 1]
714
+
715
+ # Optional: Double-check dtype here if needed, but should be handled by detokenize_audio now
716
+ # if output_wav.dtype != np.float32:
717
+ # logger.warning(f"Audio dtype after detokenize is {output_wav.dtype}. Converting to float32.")
718
+ # output_wav = output_wav.astype(np.float32)
719
+ # output_wav = np.clip(output_wav, -1.0, 1.0) # Clipping done in detokenize_audio
720
+
721
+ except Exception as e:
722
+ logger.error(f"Error during audio detokenization: {e}")
723
+ import traceback
724
+ traceback.print_exc()
725
+ raise RuntimeError("Audio detokenization failed.") from e
726
+
727
+ return {"audio": output_wav, "sampling_rate": self.sampling_rate}
728
+
729
+
730
+ @classmethod
731
+ def from_pretrained(
732
+ cls,
733
+ pretrained_model_name_or_path: Union[str, os.PathLike],
734
+ cache_dir: Optional[Union[str, os.PathLike]] = None,
735
+ force_download: bool = False,
736
+ local_files_only: bool = False,
737
+ token: Optional[Union[str, bool]] = None,
738
+ revision: str = "main",
739
+ trust_remote_code: bool = False, # Allow passing this, needed for config potentially
740
+ **kwargs,
741
+ ):
742
+ r"""
743
+ Instantiate a SparkTTSProcessor from pretrained components.
744
+ """
745
+ # Pop specific kwargs for this method
746
+ config = kwargs.pop("config", None) # Allow passing config explicitly
747
+
748
+ # --- 1. Load Config (to find component paths) ---
749
+ # We need the config even if the processor doesn't store it permanently,
750
+ # just to find where the tokenizer/feature_extractor live.
751
+ loaded_config = None
752
+ if not isinstance(config, SparkTTSConfig):
753
+ try:
754
+ # Load the specific config class
755
+ loaded_config = SparkTTSConfig.from_pretrained(
756
+ pretrained_model_name_or_path,
757
+ cache_dir=cache_dir,
758
+ force_download=force_download,
759
+ local_files_only=local_files_only,
760
+ token=token,
761
+ revision=revision,
762
+ trust_remote_code=trust_remote_code, # Config might be custom
763
+ **kwargs, # Pass relevant kwargs
764
+ )
765
+ except Exception as e:
766
+ logger.warning(
767
+ f"Could not load SparkTTSConfig from {pretrained_model_name_or_path}. "
768
+ f"Attempting to load components from default relative paths ('LLM', 'wav2vec2-large-xlsr-53'). Error: {e}"
769
+ )
770
+ loaded_config = None # Fallback
771
+ else:
772
+ # Config object was passed directly
773
+ loaded_config = config
774
+
775
+
776
+ # --- 2. Determine Component Paths ---
777
+ llm_tokenizer_path_or_id = "./LLM" # Default relative path
778
+ w2v_processor_path_or_id = "./wav2vec2-large-xlsr-53" # Default relative path
779
+
780
+ if loaded_config:
781
+ llm_tokenizer_path_or_id = getattr(loaded_config, 'llm_model_name_or_path', llm_tokenizer_path_or_id)
782
+ w2v_processor_path_or_id = getattr(loaded_config, 'wav2vec2_model_name_or_path', w2v_processor_path_or_id)
783
+
784
+ # The component `from_pretrained` methods handle resolving these paths/IDs
785
+ # whether they are relative subfolders of `pretrained_model_name_or_path`
786
+ # or separate Hub IDs.
787
+
788
+ # --- 3. Load Components ---
789
+ # Pass down relevant kwargs for loading components
790
+ component_loading_kwargs = {
791
+ "cache_dir": cache_dir,
792
+ "force_download": force_download,
793
+ "local_files_only": local_files_only,
794
+ "token": token,
795
+ "revision": revision,
796
+ **kwargs # Pass other user kwargs
797
+ }
798
+ try:
799
+ # Tokenizer might require trust_remote_code if its class is custom
800
+ tokenizer = AutoTokenizer.from_pretrained(
801
+ pretrained_model_name_or_path, # Main path
802
+ subfolder=llm_tokenizer_path_or_id.lstrip('./'), # Specify subfolder relative to main path
803
+ trust_remote_code=trust_remote_code,
804
+ **component_loading_kwargs
805
+ )
806
+ except Exception as e:
807
+ # Fallback: try loading directly using the path/id from config if different
808
+ if llm_tokenizer_path_or_id != "./LLM":
809
+ try:
810
+ logger.info(f"Retrying tokenizer load directly from: {llm_tokenizer_path_or_id}")
811
+ tokenizer = AutoTokenizer.from_pretrained(
812
+ llm_tokenizer_path_or_id,
813
+ trust_remote_code=trust_remote_code,
814
+ **component_loading_kwargs
815
+ )
816
+ except Exception as e2:
817
+ raise OSError(f"Could not load tokenizer using main path + subfolder or directly from '{llm_tokenizer_path_or_id}'. Error: {e2}") from e
818
+ else:
819
+ raise OSError(f"Could not load tokenizer from subfolder '{llm_tokenizer_path_or_id}' within '{pretrained_model_name_or_path}'. Error: {e}")
820
+
821
+
822
+ try:
823
+ # Feature extractor usually doesn't need trust_remote_code
824
+ feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
825
+ pretrained_model_name_or_path, # Main path
826
+ subfolder=w2v_processor_path_or_id.lstrip('./'), # Specify subfolder relative to main path
827
+ **component_loading_kwargs
828
+ )
829
+ except Exception as e:
830
+ # Fallback: try loading directly using the path/id from config if different
831
+ if w2v_processor_path_or_id != "./wav2vec2-large-xlsr-53":
832
+ try:
833
+ logger.info(f"Retrying feature extractor load directly from: {w2v_processor_path_or_id}")
834
+ feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
835
+ w2v_processor_path_or_id,
836
+ **component_loading_kwargs
837
+ )
838
+ except Exception as e2:
839
+ raise OSError(f"Could not load feature extractor using main path + subfolder or directly from '{w2v_processor_path_or_id}'. Error: {e2}") from e
840
+ else:
841
+ raise OSError(f"Could not load feature extractor from subfolder '{w2v_processor_path_or_id}' within '{pretrained_model_name_or_path}'. Error: {e}")
842
+
843
+
844
+ # --- 4. Instantiate processor ---
845
+ # Pass the potentially loaded config object (or None)
846
+ return cls(tokenizer=tokenizer, feature_extractor=feature_extractor, config=loaded_config)
847
+
848
+
849
+ def save_pretrained(
850
+ self,
851
+ save_directory: Union[str, os.PathLike],
852
+ push_to_hub: bool = False,
853
+ **kwargs,
854
+ ):
855
+ """
856
+ Save the processor's state (tokenizer and feature extractor files) to a directory.
857
+
858
+ Args:
859
+ save_directory (`str` or `os.PathLike`):
860
+ Directory where the processor files will be saved.
861
+ push_to_hub (`bool`, *optional*, defaults to `False`):
862
+ Whether or not to push your model to the Hugging Face Hub after saving it.
863
+ **kwargs:
864
+ Additional key word arguments passed along to the `push_to_hub` method.
865
+ """
866
+ save_directory = Path(save_directory)
867
+ save_directory.mkdir(parents=True, exist_ok=True)
868
+
869
+ # Save tokenizer
870
+ self.tokenizer.save_pretrained(str(save_directory), **kwargs)
871
+
872
+ # Save feature extractor
873
+ self.feature_extractor.save_pretrained(str(save_directory), **kwargs)
874
+
875
+ # Save the main processor config (if it exists and has relevant info)
876
+ # Note: The SparkTTSConfig is usually saved with the *model*, not the processor.
877
+ # However, if the processor holds specific config needed for reloading *itself*,
878
+ # it could be saved here. Usually, relying on the model's config is sufficient.
879
+ # if self.config:
880
+ # self.config.save_pretrained(str(save_directory)) # Example if needed
881
+
882
+ logger.info(f"Processor components saved in {save_directory}")
883
+
884
+ if push_to_hub:
885
+ # Commit message and other hub kwargs can be passed via **kwargs
886
+ commit_message = kwargs.pop("commit_message", "Save processor")
887
+ return self.push_to_hub(save_directory, commit_message=commit_message, **kwargs)
888
+
889
+ return str(save_directory) # Return path consistent with Mixin
reference.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d6db1e038c67df75cdde9ad1e43ba05f660eebc9346a30617d9b2f3892a5b201
3
+ size 1058478
requirements.txt ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Requirements for IbrahimSalah/Arabic-TTS-Spark
2
+ # Hugging Face Inference Endpoints
3
+ # https://huggingface.co/IbrahimSalah/Arabic-TTS-Spark
4
+
5
+ # Core ML frameworks
6
+ torch>=2.0.0
7
+ torchaudio>=2.0.0
8
+ transformers>=4.40.0
9
+
10
+ # Model loading
11
+ safetensors>=0.4.0
12
+ huggingface_hub>=0.20.0
13
+
14
+ # Audio processing
15
+ soundfile>=0.12.0
16
+ soxr>=0.3.0
17
+
18
+ # Tensor operations
19
+ einops>=0.7.0
20
+ numpy>=1.24.0
21
+
22
+ # Packaging utilities
23
+ packaging>=21.0
src/figures/gradio_TTS.png ADDED
src/figures/gradio_control.png ADDED
src/figures/infer_control.png ADDED

Git LFS Details

  • SHA256: bfb1617c623cc25f5af6ce1e18331f1e0c61892e21bb90702c9ce2b2b5c0e8e2
  • Pointer size: 131 Bytes
  • Size of remote file: 127 kB
src/figures/infer_voice_cloning.png ADDED

Git LFS Details

  • SHA256: 9cb85e0217bfa18ce35dc5237c85b46fd1eeac7434b3a27c89a9136b3707d920
  • Pointer size: 131 Bytes
  • Size of remote file: 119 kB
src/logo/HKUST.jpg ADDED

Git LFS Details

  • SHA256: 65810d8063ffa90864943ff6a7aec52c0704c9230b71fc46b302732950facace
  • Pointer size: 131 Bytes
  • Size of remote file: 102 kB
src/logo/NPU.jpg ADDED

Git LFS Details

  • SHA256: 7d5af77d53b2ad6bd409c068c47ea30c8a4c23fa53e12efab77354f862625bb2
  • Pointer size: 131 Bytes
  • Size of remote file: 152 kB
src/logo/NTU.jpg ADDED
src/logo/SJU.jpg ADDED

Git LFS Details

  • SHA256: 82d9ab05b6c7ceed18348a4ad43ad147ae5607ba629265482cd049e812e65e1b
  • Pointer size: 131 Bytes
  • Size of remote file: 364 kB
src/logo/SparkAudio.jpg ADDED
src/logo/SparkAudio2.jpg ADDED
src/logo/SparkTTS.jpg ADDED
src/logo/SparkTTS.png ADDED

Git LFS Details

  • SHA256: d257dd65c174aa28e21a341906f18b242e08a859e14cc72d8b86a54ad7e096ec
  • Pointer size: 131 Bytes
  • Size of remote file: 102 kB
src/logo/mobvoi.jpg ADDED

Git LFS Details

  • SHA256: 74ac35394031e708f738fd99508f8f941f15bbcb99ea5abbc98cd8795ed44831
  • Pointer size: 131 Bytes
  • Size of remote file: 431 kB
src/logo/mobvoi.png ADDED

Git LFS Details

  • SHA256: 48af5c5a1a8077e6b8db7bfe63c249ed99bf2e6a085ecbceb728614f6a3c94ae
  • Pointer size: 131 Bytes
  • Size of remote file: 120 kB
wav2vec2-large-xlsr-53/README.md ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language: multilingual
3
+ datasets:
4
+ - common_voice
5
+ tags:
6
+ - speech
7
+ license: apache-2.0
8
+ ---
9
+
10
+ # Wav2Vec2-XLSR-53
11
+
12
+ [Facebook's XLSR-Wav2Vec2](https://ai.facebook.com/blog/wav2vec-20-learning-the-structure-of-speech-from-raw-audio/)
13
+
14
+ The base model pretrained on 16kHz sampled speech audio. When using the model make sure that your speech input is also sampled at 16Khz. Note that this model should be fine-tuned on a downstream task, like Automatic Speech Recognition. Check out [this blog](https://huggingface.co/blog/fine-tune-wav2vec2-english) for more information.
15
+
16
+ [Paper](https://arxiv.org/abs/2006.13979)
17
+
18
+ Authors: Alexis Conneau, Alexei Baevski, Ronan Collobert, Abdelrahman Mohamed, Michael Auli
19
+
20
+ **Abstract**
21
+ This paper presents XLSR which learns cross-lingual speech representations by pretraining a single model from the raw waveform of speech in multiple languages. We build on wav2vec 2.0 which is trained by solving a contrastive task over masked latent speech representations and jointly learns a quantization of the latents shared across languages. The resulting model is fine-tuned on labeled data and experiments show that cross-lingual pretraining significantly outperforms monolingual pretraining. On the CommonVoice benchmark, XLSR shows a relative phoneme error rate reduction of 72% compared to the best known results. On BABEL, our approach improves word error rate by 16% relative compared to a comparable system. Our approach enables a single multilingual speech recognition model which is competitive to strong individual models. Analysis shows that the latent discrete speech representations are shared across languages with increased sharing for related languages. We hope to catalyze research in low-resource speech understanding by releasing XLSR-53, a large model pretrained in 53 languages.
22
+
23
+ The original model can be found under https://github.com/pytorch/fairseq/tree/master/examples/wav2vec#wav2vec-20.
24
+
25
+ # Usage
26
+
27
+ See [this notebook](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/Fine_Tune_XLSR_Wav2Vec2_on_Turkish_ASR_with_%F0%9F%A4%97_Transformers.ipynb) for more information on how to fine-tune the model.
28
+
29
+ ![model image](https://raw.githubusercontent.com/patrickvonplaten/scientific_images/master/xlsr_wav2vec2.png)
wav2vec2-large-xlsr-53/config.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_dropout": 0.0,
3
+ "apply_spec_augment": true,
4
+ "architectures": [
5
+ "Wav2Vec2ForPreTraining"
6
+ ],
7
+ "attention_dropout": 0.1,
8
+ "bos_token_id": 1,
9
+ "codevector_dim": 768,
10
+ "contrastive_logits_temperature": 0.1,
11
+ "conv_bias": true,
12
+ "conv_dim": [
13
+ 512,
14
+ 512,
15
+ 512,
16
+ 512,
17
+ 512,
18
+ 512,
19
+ 512
20
+ ],
21
+ "conv_kernel": [
22
+ 10,
23
+ 3,
24
+ 3,
25
+ 3,
26
+ 3,
27
+ 2,
28
+ 2
29
+ ],
30
+ "conv_stride": [
31
+ 5,
32
+ 2,
33
+ 2,
34
+ 2,
35
+ 2,
36
+ 2,
37
+ 2
38
+ ],
39
+ "ctc_loss_reduction": "sum",
40
+ "ctc_zero_infinity": false,
41
+ "diversity_loss_weight": 0.1,
42
+ "do_stable_layer_norm": true,
43
+ "eos_token_id": 2,
44
+ "feat_extract_activation": "gelu",
45
+ "feat_extract_dropout": 0.0,
46
+ "feat_extract_norm": "layer",
47
+ "feat_proj_dropout": 0.1,
48
+ "feat_quantizer_dropout": 0.0,
49
+ "final_dropout": 0.0,
50
+ "gradient_checkpointing": false,
51
+ "hidden_act": "gelu",
52
+ "hidden_dropout": 0.1,
53
+ "hidden_size": 1024,
54
+ "initializer_range": 0.02,
55
+ "intermediate_size": 4096,
56
+ "layer_norm_eps": 1e-05,
57
+ "layerdrop": 0.1,
58
+ "mask_channel_length": 10,
59
+ "mask_channel_min_space": 1,
60
+ "mask_channel_other": 0.0,
61
+ "mask_channel_prob": 0.0,
62
+ "mask_channel_selection": "static",
63
+ "mask_feature_length": 10,
64
+ "mask_feature_prob": 0.0,
65
+ "mask_time_length": 10,
66
+ "mask_time_min_space": 1,
67
+ "mask_time_other": 0.0,
68
+ "mask_time_prob": 0.075,
69
+ "mask_time_selection": "static",
70
+ "model_type": "wav2vec2",
71
+ "num_attention_heads": 16,
72
+ "num_codevector_groups": 2,
73
+ "num_codevectors_per_group": 320,
74
+ "num_conv_pos_embedding_groups": 16,
75
+ "num_conv_pos_embeddings": 128,
76
+ "num_feat_extract_layers": 7,
77
+ "num_hidden_layers": 24,
78
+ "num_negatives": 100,
79
+ "pad_token_id": 0,
80
+ "proj_codevector_dim": 768,
81
+ "transformers_version": "4.7.0.dev0",
82
+ "vocab_size": 32
83
+ }
wav2vec2-large-xlsr-53/preprocessor_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "do_normalize": true,
3
+ "feature_extractor_type": "Wav2Vec2FeatureExtractor",
4
+ "feature_size": 1,
5
+ "padding_side": "right",
6
+ "padding_value": 0,
7
+ "return_attention_mask": true,
8
+ "sampling_rate": 16000
9
+ }
wav2vec2-large-xlsr-53/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:314340227371a608f71adcd5f0de5933824fe77e55822aa4b24dba9c1c364dcb
3
+ size 1269737156