Orpheus (tr)

.gitattributes

@@ -190,3 +190,5 @@ en/Orpheus-Liam-GGUF/Orpheus-Liam.Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
 en/Orpheus-Liam/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 tr/Orpheus-KhanAcademy-TR-Math/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 tr/Orpheus-KhanAcademy-TR/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tr/Orpheus-TTS-Turkish-PT-2000-GGUF/orpheus-tts-turkish-pt-2000.gguf filter=lfs diff=lfs merge=lfs -text
+tr/Orpheus-TTS-Turkish-PT-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

tr/Orpheus-TTS-Turkish-PT-2000-GGUF/.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+orpheus-2.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
+orpheus-tts-turkish-pt-2000.gguf filter=lfs diff=lfs merge=lfs -text

tr/Orpheus-TTS-Turkish-PT-2000-GGUF/README.md

@@ -0,0 +1,12 @@
+---
+license: mit
+language:
+- tr
+base_model:
+- Karayakar/Orpheus-TTS-Turkish-PT-2000
+pipeline_tag: text-to-speech
+tags:
+- Turkish
+datasets:
+- Karayakar/OrpheusTTS110K-TR-Data
+---

tr/Orpheus-TTS-Turkish-PT-2000-GGUF/orpheus-tts-turkish-pt-2000.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cad772a1af61cf106651cd7345211929b97444157d99ec8522194c0ff7c2a338
+size 6610828992

tr/Orpheus-TTS-Turkish-PT-2000-GGUF/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/Karayakar/Orpheus-TTS-Turkish-PT-2000-GGUF

tr/Orpheus-TTS-Turkish-PT-2000/.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

tr/Orpheus-TTS-Turkish-PT-2000/README.md

@@ -0,0 +1,385 @@
+---
+license: mit
+language:
+- tr
+base_model:
+- canopylabs/orpheus-3b-0.1-pretrained
+tags:
+- karayakar
+- Turkish
+- Turkce
+- TTS
+- Orpheus
+- Text-to-Speech
+pipeline_tag: text-to-speech
+---
+
+# Orpheus TTS Turkish Model
+
+Orpheus TTS Turkish Pretrain (step 2000) 
+model is trained based on "canopylabs/orpheus-3b-0.1-pretrained".
+
+Syntethic voice data over 60 hrs used for initial training.
++160hrs additional Syntethic voice data mixed in training.
+400 Emoji (real voice) data used for emoji support. 
+
+you can interact with the model - Flask API
+
+
+# Emotion Support 
+
+Model supports below emotions in the text.
+```
+<laugh> – gülme
+
+<chuckle> – kıkırdama
+
+<sigh> – iç çekme
+
+<cough> – öksürme
+
+<sniffle> – <burnunu çekme>
+
+<groan> – inleme
+
+<yawn> – esneme
+
+<gasp> – nefesi kesilme / şaşkınlıkla soluma
+```
+
+
+# API 
+
+Flask configured to run on port 5400 (you can change in the below script)
+
+```
+POST http://127.0.0.1:5400/generate HTTP/1.1
+User-Agent: Fiddler
+content-type: application/json
+Host: 127.0.0.1:5400
+Content-Length: 110
+
+{
+    "text": "Merhaba, orpheusTTS Turkce deneme"
+}
+
+```
+
+# Create Environment
+
+windows:
+```
+#create virtual environment
+python -m venv venv 
+venv\Scripts\activate
+
+python inference.py
+
+
+
+```
+
+# Training
+
+```
+For training with your own data, you can check
+train.py
+config.yaml
+
+```
+
+
+
+
+
+# inference.py
+(please install the necessary libraries)
+
+```
+# respective torch from https://pytorch.org/
+pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124
+pip install snac pathlib torch transformers huggingface_hub librosa numpy scipy torchaudio Flask jsonify
+
+
+```
+
+```
+import os
+from snac import SNAC
+from pathlib import Path
+import torch
+from transformers import AutoModelForCausalLM, Trainer, TrainingArguments, AutoTokenizer,BitsAndBytesConfig
+from huggingface_hub import snapshot_download
+import librosa
+import numpy as np
+from scipy.io.wavfile import write
+import torchaudio
+from flask import Flask, jsonify, request
+
+modelLocalPath="D:\\...\\Karayakar\\Orpheus-TTS-Turkish-PT-2000"
+
+
+def load_orpheus_tokenizer(model_id: str = modelLocalPath) -> AutoTokenizer:
+    tokenizer = AutoTokenizer.from_pretrained(model_id,local_files_only=True, device_map="cuda")
+    return tokenizer
+
+def load_snac():
+    snac_model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz")
+    return snac_model
+
+def load_orpheus_auto_model(model_id: str = modelLocalPath):
+    model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16,local_files_only=True, device_map="cuda")
+    model.cuda()
+    return model
+
+
+
+def tokenize_audio(audio_file_path, snac_model):
+    audio_array, sample_rate = librosa.load(audio_file_path, sr=24000)
+    waveform = torch.from_numpy(audio_array).unsqueeze(0)
+    waveform = waveform.to(dtype=torch.float32)
+
+    waveform = waveform.unsqueeze(0)
+
+    with torch.inference_mode():
+        codes = snac_model.encode(waveform)
+
+    all_codes = []
+    for i in range(codes[0].shape[1]):
+        all_codes.append(codes[0][0][i].item() + 128266)
+        all_codes.append(codes[1][0][2 * i].item() + 128266 + 4096)
+        all_codes.append(codes[2][0][4 * i].item() + 128266 + (2 * 4096))
+        all_codes.append(codes[2][0][(4 * i) + 1].item() + 128266 + (3 * 4096))
+        all_codes.append(codes[1][0][(2 * i) + 1].item() + 128266 + (4 * 4096))
+        all_codes.append(codes[2][0][(4 * i) + 2].item() + 128266 + (5 * 4096))
+        all_codes.append(codes[2][0][(4 * i) + 3].item() + 128266 + (6 * 4096))
+
+    return all_codes
+
+
+def prepare_inputs(
+    fpath_audio_ref,
+    audio_ref_transcript: str,
+    text_prompts: list[str],
+    snac_model,
+    tokenizer,
+):
+
+    
+    start_tokens = torch.tensor([[128259]], dtype=torch.int64)
+    end_tokens = torch.tensor([[128009, 128260, 128261, 128257]], dtype=torch.int64)
+    final_tokens = torch.tensor([[128258, 128262]], dtype=torch.int64)
+
+
+    all_modified_input_ids = []
+    for prompt in text_prompts:
+        input_ids = tokenizer(prompt, return_tensors="pt").input_ids
+        #second_input_ids = torch.cat([zeroprompt_input_ids, start_tokens, input_ids, end_tokens], dim=1)
+        second_input_ids = torch.cat([start_tokens, input_ids, end_tokens], dim=1)
+        all_modified_input_ids.append(second_input_ids)
+
+    all_padded_tensors = []
+    all_attention_masks = []
+    max_length = max([modified_input_ids.shape[1] for modified_input_ids in all_modified_input_ids])
+
+    for modified_input_ids in all_modified_input_ids:
+        padding = max_length - modified_input_ids.shape[1]
+        padded_tensor = torch.cat([torch.full((1, padding), 128263, dtype=torch.int64), modified_input_ids], dim=1)
+        attention_mask = torch.cat([torch.zeros((1, padding), dtype=torch.int64),
+                                    torch.ones((1, modified_input_ids.shape[1]), dtype=torch.int64)], dim=1)
+        all_padded_tensors.append(padded_tensor)
+        all_attention_masks.append(attention_mask)
+
+    all_padded_tensors = torch.cat(all_padded_tensors, dim=0)
+    all_attention_masks = torch.cat(all_attention_masks, dim=0)
+
+    input_ids = all_padded_tensors.to("cuda")
+    attention_mask = all_attention_masks.to("cuda")
+    return input_ids, attention_mask
+
+
+
+def inference(model, input_ids, attention_mask):
+    with torch.no_grad():
+        generated_ids = model.generate(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            max_new_tokens=2048,
+            do_sample=True,
+            temperature=0.2,
+            top_k=10,
+            top_p=0.9,
+            repetition_penalty=1.9,
+            num_return_sequences=1,
+            eos_token_id=128258,
+
+        )
+
+        generated_ids = torch.cat([generated_ids, torch.tensor([[128262]]).to("cuda")], dim=1) # EOAI
+
+        return generated_ids
+
+
+def convert_tokens_to_speech(generated_ids, snac_model):
+    token_to_find = 128257
+    token_to_remove = 128258
+    token_indices = (generated_ids == token_to_find).nonzero(as_tuple=True)
+
+    if len(token_indices[1]) > 0:
+        last_occurrence_idx = token_indices[1][-1].item()
+        cropped_tensor = generated_ids[:, last_occurrence_idx + 1:]
+    else:
+        cropped_tensor = generated_ids
+
+    _mask = cropped_tensor != token_to_remove
+    processed_rows = []
+    for row in cropped_tensor:
+        masked_row = row[row != token_to_remove]
+        processed_rows.append(masked_row)
+
+    code_lists = []
+    for row in processed_rows:
+        row_length = row.size(0)
+        new_length = (row_length // 7) * 7 
+        trimmed_row = row[:new_length]
+        trimmed_row = [t - 128266 for t in trimmed_row]
+        code_lists.append(trimmed_row)
+
+    my_samples = []
+    for code_list in code_lists:
+        samples = redistribute_codes(code_list, snac_model)
+        my_samples.append(samples)
+
+    return my_samples
+
+
+def redistribute_codes(code_list, snac_model):
+    layer_1 = []
+    layer_2 = []
+    layer_3 = []
+
+    for i in range((len(code_list) + 1) // 7):
+        layer_1.append(code_list[7 * i])
+        layer_2.append(code_list[7 * i + 1] - 4096)
+        layer_3.append(code_list[7 * i + 2] - (2 * 4096))
+        layer_3.append(code_list[7 * i + 3] - (3 * 4096))
+        layer_2.append(code_list[7 * i + 4] - (4 * 4096))
+        layer_3.append(code_list[7 * i + 5] - (5 * 4096))
+        layer_3.append(code_list[7 * i + 6] - (6 * 4096))
+
+    codes = [
+        torch.tensor(layer_1).unsqueeze(0),
+        torch.tensor(layer_2).unsqueeze(0),
+        torch.tensor(layer_3).unsqueeze(0)
+    ]
+    audio_hat = snac_model.decode(codes)
+    return audio_hat 
+
+
+def to_wav_from(samples: list) -> list[np.ndarray]:
+    """Converts a list of PyTorch tensors (or NumPy arrays) to NumPy arrays."""
+    processed_samples = []
+
+    for s in samples:
+        if isinstance(s, torch.Tensor):  
+            s = s.detach().squeeze().to('cpu').numpy()
+        else:  
+            s = np.squeeze(s)
+
+        processed_samples.append(s)
+
+    return processed_samples
+
+
+def zero_shot_tts(fpath_audio_ref, audio_ref_transcript, texts: list[str], model, snac_model, tokenizer):
+    print(f"fpath_audio_ref {fpath_audio_ref}")
+    print(f"audio_ref_transcript {audio_ref_transcript}")
+    print(f"texts {texts}")
+    inp_ids, attn_mask = prepare_inputs(fpath_audio_ref, audio_ref_transcript, texts, snac_model, tokenizer)
+    print(f"input_id_len:{len(inp_ids)}")
+    gen_ids = inference(model, inp_ids, attn_mask)
+    samples = convert_tokens_to_speech(gen_ids, snac_model)
+    wav_forms = to_wav_from(samples)
+    return wav_forms
+
+
+def save_wav(samples: list[np.array], sample_rate: int, filenames: list[str]):
+    """ Saves a list of tensors as .wav files.
+
+    Args:
+        samples (list[torch.Tensor]): List of audio tensors.
+        sample_rate (int): Sample rate in Hz.
+        filenames (list[str]): List of filenames to save.
+    """
+    wav_data = to_wav_from(samples)
+
+    for data, filename in zip(wav_data, filenames):
+        write(filename, sample_rate, data.astype(np.float32))
+        print(f"saved to {filename}")
+
+
+def get_ref_audio_and_transcript(root_folder: str):
+    root_path = Path(root_folder)
+    print(f"root_path   {root_path}")
+    out = []
+    for speaker_folder in root_path.iterdir():
+        if speaker_folder.is_dir():  # Ensure it's a directory
+            wav_files = list(speaker_folder.glob("*.wav"))
+            txt_files = list(speaker_folder.glob("*.txt"))
+
+            if wav_files and txt_files:
+                ref_audio = wav_files[0]  # Assume only one .wav file per folder
+                transcript = txt_files[0].read_text(encoding="utf-8").strip()
+                out.append((ref_audio, transcript))
+
+    return out
+
+app = Flask(__name__)
+
+
+@app.route('/generate', methods=['POST'])
+def generate():
+    content = request.json
+    process_data(content)
+    rresponse = {
+        'received': content,
+        'status': 'success'
+    }
+    response= jsonify(rresponse)
+    response.headers['Content-Type'] = 'application/json; charset=utf-8'
+    return response
+
+
+
+def process_data(jsonText):
+    texts = [f"{jsonText['text']}"]
+    #print(f"texts:{texts}")
+    #print(f"prompt_pairs:{prompt_pairs}")
+    for fpath_audio, audio_transcript in prompt_pairs:
+        print(f"zero shot: {fpath_audio} {audio_transcript}")
+        wav_forms = zero_shot_tts(fpath_audio, audio_transcript, texts, model, snac_model, tokenizer)
+
+        import os
+        from pathlib import Path
+        from datetime import datetime
+        out_dir = Path(fpath_audio).parent / "inference"
+        #print(f"out_dir:{out_dir}")
+        out_dir.mkdir(parents=True, exist_ok=True)  #
+        timestamp_str = str(int(datetime.now().timestamp()))
+        file_names = [f"{out_dir.as_posix()}/{Path(fpath_audio).stem}_{i}_{timestamp_str}.wav" for i, t in enumerate(texts)]
+        #print(f"file_names:{file_names}")
+        save_wav(wav_forms, 24000, file_names)
+        
+      
+
+if __name__ == "__main__":
+    tokenizer = load_orpheus_tokenizer()
+    model = load_orpheus_auto_model()
+    snac_model = load_snac()
+    prompt_pairs = get_ref_audio_and_transcript("D:\\AI_APPS\\Orpheus-TTS\\data")
+    print(f"snac_model loaded")
+    app.run(debug=True,port=5400)
+   
+
+
+```

tr/Orpheus-TTS-Turkish-PT-2000/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "G:\\OPENAI\\Orpheus-TTS\\pretrain\\checkpoints_Orpheus_TTS_KA_60HRS_24000Khz\\checkpoint-500",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.49.0",
+  "use_cache": true,
+  "vocab_size": 156940
+}

tr/Orpheus-TTS-Turkish-PT-2000/config.yaml

@@ -0,0 +1,23 @@
+# Model
+model_name: "Karayakar/Orpheus-TTS-Turkish-PT-2000"
+tokenizer_name: "Karayakar/Orpheus-TTS-Turkish-PT-2000"
+ 
+
+# Training Args
+epochs: 1
+batch_size: 1
+number_processes: 1
+pad_token: 128263
+save_steps: 100
+learning_rate: 5.0e-5
+ratio: 1:1
+
+# Datasets
+text_QA_dataset: "your train dataset"
+TTS_dataset: "your train dataset"
+
+# Naming and paths
+save_folder: "checkpoints_Orpheus_TTS"
+project_name: "pretrain-orpheus"
+run_name: "pretrain-orpheus"
+#accelerate launch train.py

tr/Orpheus-TTS-Turkish-PT-2000/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.49.0"
+}

tr/Orpheus-TTS-Turkish-PT-2000/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61d4ae63f446c953ff136785c60538c47ddc30a6e0eaaa41895e8225071b1c80
+size 4948557560

tr/Orpheus-TTS-Turkish-PT-2000/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c041d2ac07dc3e428cbe9bc5882bce45fc3a83a05d26e93411b3ffc772562fe8
+size 4932808960

tr/Orpheus-TTS-Turkish-PT-2000/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bbcde72de8c320bf456ca24c68e8cd3ce375d76d214415b32fe064422026599
+size 3322130968

tr/Orpheus-TTS-Turkish-PT-2000/model.safetensors.index.json

@@ -0,0 +1,261 @@
+{
+  "metadata": {
+    "total_size": 13203468288
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

tr/Orpheus-TTS-Turkish-PT-2000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95a013621fb4d350dab6ace0142f9e142286c68602c0f0a6438d3d1e3b4419f7
+size 26407159030

tr/Orpheus-TTS-Turkish-PT-2000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea005b38dffb3ddb68fe35f651e4ee5e119bf913293dda58efdd7f56f4335656
+size 14244

tr/Orpheus-TTS-Turkish-PT-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da762999d2c4127b623e631487e702dde005ba1214253149c024342697124c4f
+size 1064

tr/Orpheus-TTS-Turkish-PT-2000/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/Karayakar/Orpheus-TTS-Turkish-PT-2000

tr/Orpheus-TTS-Turkish-PT-2000/special_tokens_map.json

@@ -0,0 +1,26 @@
+{
+  "additional_special_tokens": [
+    "<|audio|>"
+  ],
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|finetune_right_pad_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tr/Orpheus-TTS-Turkish-PT-2000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc3fecb199b4170636dbfab986d25f628157268d37b861f9cadaca60b1353bce
+size 22849547

tr/Orpheus-TTS-Turkish-PT-2000/train.py

@@ -0,0 +1,236 @@
+import torch
+from datasets import load_dataset,load_from_disk
+from transformers import AutoModelForCausalLM, Trainer, TrainingArguments, AutoTokenizer
+import numpy as np
+from torch.distributed.fsdp.fully_sharded_data_parallel import FullStateDictConfig
+from torch.distributed.fsdp import (
+    FullyShardedDataParallel as FSDP, FullStateDictConfig, StateDictType)
+from torch.utils.data import DataLoader, Dataset
+from torch.utils.data.distributed import DistributedSampler
+import yaml
+import wandb
+from huggingface_hub import HfApi
+
+config_file = "config.yaml"
+
+with open(config_file, "r") as file:
+    config = yaml.safe_load(file)
+
+dsn1 = config["text_QA_dataset"]
+dsn2 = config["TTS_dataset"]
+
+model_name = config["model_name"]
+tokenizer_name = config["tokenizer_name"]
+
+run_name = config["run_name"]
+project_name = config["project_name"]
+base_repo_id = config["save_folder"]
+
+epochs = config["epochs"]
+batch_size = config["batch_size"]
+save_steps = config["save_steps"]
+pad_token = config["pad_token"]
+number_processes = config["number_processes"]
+learning_rate = config["learning_rate"]
+config_ratio = config["ratio"]
+
+
+
+
+class BatchedRatioDataset(Dataset):
+    def __init__(self, dataset1, dataset2, batch_total, ratio=config_ratio):
+        self.dataset1 = dataset1
+        self.dataset2 = dataset2
+        self.batch_total = batch_total
+        self.ratio = ratio  
+
+        num_cycles_ds1 = len(dataset1) // (batch_total * ratio)
+        num_cycles_ds2 = len(dataset2) // batch_total
+        self.num_cycles = min(num_cycles_ds1, num_cycles_ds2)
+
+        self.length = self.num_cycles * (ratio + 1) * batch_total
+
+    def __len__(self):
+        print("accessing length", self.length)
+        return int(self.length)
+
+    def __getitem__(self, index):
+        # Compute the cycle length in terms of samples.
+        cycle_length = (self.ratio + 1) * self.batch_total
+        cycle = index // cycle_length
+        pos_in_cycle = index % cycle_length
+
+        if pos_in_cycle < self.ratio * self.batch_total:
+            batch_in_cycle = pos_in_cycle // self.batch_total
+            sample_in_batch = pos_in_cycle % self.batch_total
+            ds1_index = cycle * self.ratio * self.batch_total + batch_in_cycle * self.batch_total + sample_in_batch
+            return self.dataset1[ds1_index]
+        else:
+            # We are in the dataset2 batch for this cycle.
+            sample_in_batch = pos_in_cycle - self.ratio * self.batch_total
+            ds2_index = cycle * self.batch_total + sample_in_batch
+            return self.dataset2[ds2_index]
+
+
+
+class AlternatingDistributedSampler(DistributedSampler):
+    def __init__(self, dataset, num_replicas=None, rank=None, shuffle=False):
+        super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle)
+        self.shuffle = shuffle
+
+    def __iter__(self):
+        indices = list(range(len(self.dataset)))
+        indices = indices[self.rank:self.total_size:self.num_replicas]
+        return iter(indices)
+
+
+class FSDPTrainer(Trainer):
+    def __init__(self, *args, log_ratio=config_ratio, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.repo_id = base_repo_id
+        self.api = HfApi()
+
+        self.log_ratio = log_ratio
+        self.text_step  = 0
+        self.audio_step = 0
+
+    def get_train_dataloader(self):
+        sampler = AlternatingDistributedSampler(
+            self.train_dataset,
+            num_replicas=torch.distributed.get_world_size(),
+            rank=torch.distributed.get_rank(),
+            shuffle=False,
+        )
+
+        return DataLoader(
+            self.train_dataset,
+            batch_size=self.args.per_device_train_batch_size,
+            sampler=sampler,
+            collate_fn=self.data_collator,
+            drop_last=self.args.dataloader_drop_last,
+            num_workers=0,
+            pin_memory=self.args.dataloader_pin_memory,
+        )
+
+    def log(self, logs, start_time=None):
+        super().log(logs, start_time)
+        if self.is_world_process_zero():
+            global_step = self.state.global_step
+            # Each cycle is (log_ratio + 1) steps: first log_ratio steps for text_loss, then one for audio_loss.
+            cycle_length = self.log_ratio + 1
+            if (global_step % cycle_length) + self.log_ratio - 1 < self.log_ratio:
+                wandb.log({"audio_loss": logs["loss"], "audio_step": self.audio_step})
+                self.audio_step += 1
+            else:
+                wandb.log({"text_loss": logs["loss"], "text_step": self.text_step})
+                self.text_step += 1
+
+    def save_model(self, output_dir=None, _internal_call=False):
+        if output_dir is None:
+            output_dir = self.args.output_dir
+        self.save_and_push_model(output_dir)
+
+    def save_and_push_model(self, output_dir):
+        save_policy = FullStateDictConfig(offload_to_cpu=True, rank0_only=True)
+        with FSDP.state_dict_type(self.model, StateDictType.FULL_STATE_DICT, save_policy):
+            cpu_state_dict = self.model.state_dict()
+        self.model.save_pretrained(output_dir, state_dict=cpu_state_dict)
+
+
+def data_collator(features):
+    # max_length = 2656 # set a crop based on vram - ideally you have stacked all sequences to the same length
+    # from 3b on 8 h100s fsdp, at bf16, 8192 works well.
+    input_ids = [f["input_ids"] for f in features]
+
+    if any("attention_mask" not in f for f in features):
+        attention_mask = [[1]*len(ids) for ids in input_ids]
+    else:
+        attention_mask = [f["attention_mask"] for f in features]
+
+    if any("labels" not in f for f in features):
+        labels = input_ids
+    else:
+        labels = [f["labels"] for f in features]
+
+    input_ids = torch.nn.utils.rnn.pad_sequence([torch.tensor(
+        i, dtype=torch.long) for i in input_ids], batch_first=True, padding_value=pad_token)
+    attention_mask = torch.nn.utils.rnn.pad_sequence([torch.tensor(
+        m, dtype=torch.long) for m in attention_mask], batch_first=True, padding_value=0)
+    labels = torch.nn.utils.rnn.pad_sequence([torch.tensor(
+        l, dtype=torch.long) for l in labels], batch_first=True, padding_value=-100)
+
+    return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}
+
+#wandb offline
+#wandb.init(project=project_name, name=run_name)
+
+
+tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
+model = AutoModelForCausalLM.from_pretrained(model_name, local_files_only=True)
+#attn_implementation="flash_attention_2",torch_dtype=torch.bfloat16,
+model.cuda()
+
+number_add_tokens = 7 * 4096 + 10
+new_tokens = [f"<custom_token_{i}>" for i in range(0, number_add_tokens + 1)]
+tokenizer.add_tokens(new_tokens)
+model.resize_token_embeddings(len(tokenizer))
+
+
+ds1 = load_from_disk(dsn1, keep_in_memory=True)# split="train")
+ds2 = load_from_disk(dsn2, keep_in_memory=True)# split="train")
+
+
+batch_total = batch_size * number_processes
+train_dataset = BatchedRatioDataset(ds1, ds2, batch_total, ratio=config_ratio)
+
+
+#training_args = TrainingArguments(
+#    overwrite_output_dir=True,
+#    num_train_epochs=epochs,
+#    per_device_train_batch_size=batch_size,
+#    logging_steps=1,
+#    bf16=True,
+#    output_dir=f"./{base_repo_id}",
+#    #fsdp="auto_wrap",
+#    report_to="none",
+#    save_steps=save_steps,
+#    remove_unused_columns=True,
+#    learning_rate=learning_rate,
+#    lr_scheduler_type="cosine", 
+#)
+#
+#
+#trainer = FSDPTrainer(
+#    model=model,
+#    args=training_args,
+#    train_dataset=train_dataset,
+#    data_collator=data_collator,
+#    log_ratio=config_ratio
+#)
+#
+#trainer.train()
+
+
+training_args = TrainingArguments(
+    overwrite_output_dir=True,
+    num_train_epochs=epochs,
+    per_device_train_batch_size=batch_size, 
+    logging_steps=1,
+    bf16=True,
+    output_dir=f"./{base_repo_id}",
+    report_to="tensorboard", 
+    save_steps=save_steps,
+    remove_unused_columns=True, 
+    learning_rate=learning_rate,
+    lr_scheduler_type="cosine",
+)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_dataset,
+    data_collator=data_collator,
+    #log_ratio=config_ratio
+)
+
+trainer.train()

tr/Orpheus-TTS-Turkish-PT-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2e17615c0117656ff03a59eab05516ad636e4ce6df4182952bb10a0fe6b8314
+size 5368