Upload 2 files

Very, very early NAR-len model (that works enough but not good enough so far)

.gitattributes

@@ -18,3 +18,4 @@ loras/ckpt/lora-cyberpunk-takemura-r128-a128/lora.sft filter=lfs diff=lfs merge=
 loras/ckpt/lora-portal-glados-r128-a128/lora.sft filter=lfs diff=lfs merge=lfs -text
 loras/ckpt/lora-samandmax-sam-r128-a128/lora.sft filter=lfs diff=lfs merge=lfs -text
 models/ckpt/ar+nar-layerskip-llama-8/fp32.sft filter=lfs diff=lfs merge=lfs -text
+models/ckpt/nar-len-llama-8/fp32.sft filter=lfs diff=lfs merge=lfs -text

models/ckpt/nar-len-llama-8/fp32.sft

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02bd16380e556e4e4a41b3d45ee3a64bf439e6b5bd24f39a9e0c684155fd4aaf
+size 475711612

models/config.llama[nar-len].yaml

@@ -0,0 +1,169 @@
+sample_rate: 24_000
+audio_backend: "vocos"
+weights_format: sft
+experimental: True
+
+models:
+- name: "nar-len"
+  size: "full"
+  resp_levels: 8
+  tasks: 9
+  langs: 4
+  tones: 1
+  arch_type: llama
+  training: True
+  version: 5
+  attention: sdpa
+  dropout: 0.1
+  #loss_factors:
+  #  text: 0.01
+  #  prom: 0.5
+  #  resp: 1.0
+  capabilities: ["ar", "nar", "len"]
+  experimental:
+    audio_embedding_sums: True
+    split_classifiers: True
+    unified_position_ids: False
+    rvq_levels_p: [
+      0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0,
+      1, 2, 3, 4, 5, 6, 7
+    ]
+
+    masking_train_p: 1.0
+    masking_ratio_fixed: True
+    ignore_inputs_for_loss: True
+
+    cfg_cond_dropout_p: 0.1
+    cfg_prom_dropout_p: 0.05
+
+    #token_dropout_error: 0.001
+    #token_dropout_rate: 0.001
+    #layerskip: True
+    #layerskip_r: 2
+    #layerskip_e_scale: 0.1
+
+#loras:
+#- name : "lora-shodan"
+#  rank: 128
+#  alpha: 128
+#  training: True
+#  rvq_levels: []
+
+hyperparameters:
+  batch_size: 32
+  gradient_accumulation_steps: 4 # 8
+  gradient_clipping: 1.0
+  warmup_steps: 10
+
+  optimizer: Prodigy
+  learning_rate: 1.0
+  torch_optimizer: True
+  
+  scheduler: "" # ScheduleFree
+  torch_scheduler: True
+
+evaluation:
+  batch_size: 8
+  frequency: 500
+  size: 8
+  
+  kwargs:
+    max_duration: 500
+    max_steps: 25
+    ar_temperature: 1.0
+    repetition_penalty: 1.0
+    cfg_strength: 1.0
+    nar_temperature: 0.0
+
+trainer:
+  iterations: 1_000_000  
+  save_frequency: 250
+  keep_last_checkpoints: 4
+
+  resize_modules: True
+  
+  check_for_oom: False
+  gradient_checkpointing: True
+
+  weight_dtype: float16
+  amp: True
+
+  backend: deepspeed
+  deepspeed:
+    inferencing: False
+    amp: False
+    loss_scale_window: 250
+    min_loss_scale: 32768
+
+  load_webui: False
+
+inference:
+  backend: local
+  normalize: False
+
+  weight_dtype: float16
+  amp: True
+
+optimizations:
+  injects: False
+  replace: True
+
+  linear: False
+  embedding: False
+  optimizers: True
+
+  bitsandbytes: False
+  dadaptation: False
+  bitnet: False
+  fp8: False
+
+dataset:
+  speaker_name_getter: "lambda p: f'{p.parts[-3]}_{p.parts[-2]}'"
+  speaker_group_getter: "lambda p: f'{p.parts[-3]}'"
+  speaker_languages:
+    ja: [
+      "housamo",
+      "JA-"
+    ]
+    de: [
+      "DE-"
+    ]
+    fr: [
+      "FR-"
+    ]
+
+  use_hdf5: True
+  hdf5_flag: r
+  
+  use_metadata: True
+  validate: True
+
+  workers: 2
+  cache: True
+
+  duration_range: [1.0, 16.0]
+
+  prompt_max_samples: 1
+  prompt_duration_range: [1.0, 6.0]
+  prompt_similar_p: 0.825
+  prompt_similar_top_k: 6
+
+  resps_max_samples: 1
+  resps_append_p: 0.0
+
+  sample_type: path # path # speaker
+  sample_order: duration
+  sample_max_duration_batch: 120
+  sample_shuffle: True
+  retokenize_text: True
+
+  tasks_list: [
+    "tts", "tts", "tts", "tts", "tts", "tts", "tts",
+    "tts", "tts", "tts", "tts", "tts", "tts", "tts",
+    "len",
+  ] #, "stt", "tts-c", "ns", "sr" ]
+
+  training: []
+  validation: []
+  noise: []