data processing updates

speech/cosyvoice/dataset/processor.py

@@ -21,11 +21,146 @@ import torchaudio
 from torch.nn.utils.rnn import pad_sequence
 import torch.nn.functional as F
 import pyworld as pw
-
+import glob
+import os
+import json
 
 AUDIO_FORMAT_SETS = {'flac', 'mp3', 'm4a', 'ogg', 'opus', 'wav', 'wma'}
 
 
+def individual_file_opener(data, mode='train', tts_data={}):
+    """Load data from individual files instead of parquet
+    
+    Args:
+        data: Iterable[{src}] where src is either:
+            - Path to a directory containing audio files
+            - Path to a JSON index file
+        mode: 'train' or 'test'
+        tts_data: Dict for TTS mode
+        
+    Yields:
+        Dict with all required fields for training
+    """
+    for sample in data:
+        assert 'src' in sample
+        src = sample['src']
+        
+        # Determine if src is a directory or index file
+        if src.endswith('.json'):
+            # Load from index file
+            with open(src, 'r') as f:
+                index_data = json.load(f)
+                file_list = index_data.get('data', [])
+        else:
+            # Scan directory for wav files
+            wav_files = glob.glob(os.path.join(src, '*/*/*wav'))
+            if not wav_files:
+                # Try different patterns
+                wav_files = glob.glob(os.path.join(src, '**/*.wav'), recursive=True)
+            
+            file_list = []
+            for wav_path in wav_files:
+                # Check if all required files exist
+                txt_path = wav_path.replace('.wav', '.normalized.txt')
+                embedding_path = wav_path.replace('.wav', '_embedding.pt')
+                token_path = wav_path.replace('.wav', '_tokens.pt')
+                
+                if not os.path.exists(txt_path):
+                    logging.warning(f'Text file not found for {wav_path}, skipping')
+                    continue
+                
+                # Extract metadata
+                utt = os.path.basename(wav_path).replace('.wav', '')
+                spk = utt.split('_')[0]
+                
+                # Find speaker embedding
+                spk_embed_dir = os.path.join(os.path.dirname(src), 'spk_embeddings')
+                if not os.path.exists(spk_embed_dir):
+                    spk_embed_dir = os.path.join(src, 'spk_embeddings')
+                spk_embedding_path = os.path.join(spk_embed_dir, f'{spk}_embedding.pt')
+                
+                file_info = {
+                    'utt': utt,
+                    'spk': spk,
+                    'wav': wav_path,
+                    'text_path': txt_path,
+                    'embedding_path': embedding_path,
+                    'token_path': token_path,
+                    'spk_embedding_path': spk_embedding_path
+                }
+                file_list.append(file_info)
+        
+        # Process each file
+        for file_info in file_list:
+            try:
+                # Read audio data
+                with open(file_info['wav'], 'rb') as f:
+                    audio_data = f.read()
+                
+                # Read text
+                with open(file_info['text_path'], 'r') as f:
+                    text = ''.join(l.strip() for l in f.readlines())
+                
+                # Load embeddings if they exist
+                if os.path.exists(file_info['embedding_path']):
+                    utt_embedding = torch.load(file_info['embedding_path'])
+                    if isinstance(utt_embedding, torch.Tensor):
+                        utt_embedding = utt_embedding.tolist()
+                else:
+                    logging.warning(f"Utterance embedding not found: {file_info['embedding_path']}")
+                    # Create a dummy embedding
+                    utt_embedding = [0.0] * 192  # Assuming 192-dim embeddings
+                
+                # Load tokens if they exist
+                if os.path.exists(file_info['token_path']):
+                    speech_token = torch.load(file_info['token_path'])
+                    if isinstance(speech_token, torch.Tensor):
+                        speech_token = speech_token.tolist()
+                else:
+                    logging.warning(f"Token file not found: {file_info['token_path']}")
+                    speech_token = []
+                
+                # Load speaker embedding
+                if os.path.exists(file_info['spk_embedding_path']):
+                    spk_embedding = torch.load(file_info['spk_embedding_path'])
+                    if isinstance(spk_embedding, torch.Tensor):
+                        spk_embedding = spk_embedding.tolist()
+                else:
+                    logging.warning(f"Speaker embedding not found: {file_info['spk_embedding_path']}")
+                    # Use utterance embedding as fallback
+                    spk_embedding = utt_embedding
+                
+                # Build sample dict
+                sample_dict = {
+                    'utt': file_info['utt'],
+                    'spk': file_info['spk'],
+                    'audio_data': audio_data,
+                    'text': text,
+                    'text_token': [],  # Will be filled by tokenize processor
+                    'utt_embedding': utt_embedding,
+                    'spk_embedding': spk_embedding,
+                    'speech_token': speech_token,
+                    'wav': file_info['wav'],  # Keep original path for reference
+                }
+                
+                # Copy over any additional fields from the original sample
+                for key, value in sample.items():
+                    if key not in sample_dict:
+                        sample_dict[key] = value
+                
+                if mode == 'train':
+                    yield sample_dict
+                else:
+                    # For TTS mode
+                    if file_info['utt'] in tts_data:
+                        for index, tts_text in enumerate(tts_data[file_info['utt']]):
+                            yield {**sample_dict, 'tts_index': index, 'tts_text': tts_text}
+                    else:
+                        yield sample_dict
+                        
+            except Exception as ex:
+                logging.warning(f'Failed to process {file_info["wav"]}: {ex}')
+
 def parquet_opener(data, mode='train', tts_data={}):
     """ Give url or local file, return file descriptor
         Inplace operation.

speech/cosyvoice2.yaml

@@ -129,6 +129,9 @@ hifigan: !new:cosyvoice.hifigan.hifigan.HiFiGan
         !ref <mel_spec_transform1>
     ]
 
+individual_file_opener: !name:cosyvoice.dataset.processor.individual_file_opener
+
+
 # processor functions
 parquet_opener: !name:cosyvoice.dataset.processor.parquet_opener
 get_tokenizer: !name:cosyvoice.tokenizer.tokenizer.get_qwen_tokenizer
@@ -176,25 +179,11 @@ padding: !name:cosyvoice.dataset.processor.padding
 
 # dataset processor pipeline
 data_pipeline: [
-    !ref <parquet_opener>,
-    !ref <tokenize>,
-    !ref <filter>,
-    !ref <resample>,
-    !ref <compute_fbank>,
-    !ref <parse_embedding>,
-    !ref <shuffle>,
-    !ref <sort>,
-    !ref <batch>,
-    !ref <padding>,
-]
-data_pipeline_gan: [
-    !ref <parquet_opener>,
+    !ref <individual_file_opener>,
     !ref <tokenize>,
     !ref <filter>,
     !ref <resample>,
-    !ref <truncate>,
     !ref <compute_fbank>,
-    !ref <compute_f0>,
     !ref <parse_embedding>,
     !ref <shuffle>,
     !ref <sort>,