Upload 2 files

processing_cm3p.py

@@ -7,7 +7,6 @@ from pathlib import Path
 from typing import Optional, Union, IO, TypedDict
 
 import numpy as np
-import soxr
 from pandas import Series
 from slider import Beatmap, HoldNote
 from transformers import WhisperFeatureExtractor, AutoProcessor, BatchEncoding
@@ -151,6 +150,7 @@ class CM3PProcessorKwargs(CommonKwargs, CM3PBeatmapKwargs, CM3PTokenizerKwargs,
             "truncation": TruncationStrategy.LONGEST_FIRST,
             "window_length_sec": 30.0,
             "window_stride_sec": 30.0,
+            "min_window_length_sec": 1.0,
         },
         "metadata_kwargs": {
             "max_length": 128,
@@ -347,6 +347,7 @@ class CM3PProcessor(ProcessorMixin):
                 array = array.mean(axis=1)
             # Resample if the sampling rate is different from the expected one
             if s != sampling_rate:
+                import soxr
                 array = soxr.resample(array, s, sampling_rate, quality="HQ")
             audio_buffers.append(array)
 
@@ -433,6 +434,7 @@ class CM3PProcessor(ProcessorMixin):
 
         window_length_sec = beatmap_kwargs.pop("window_length_sec")
         window_stride_sec = beatmap_kwargs.pop("window_stride_sec")
+        min_window_length_sec = beatmap_kwargs.pop("min_window_length_sec", 1.0)
         max_length = beatmap_kwargs.get("max_length", 8000)
         metadata_max_length = metadata_kwargs.get("max_length", 128)
         sampling_rate = audio_kwargs["sampling_rate"]
@@ -506,7 +508,6 @@ class CM3PProcessor(ProcessorMixin):
 
                 # Loop through with sliding window
                 groups_search_index = 0
-                min_window_length_sec = 8
                 for start_sec in np.arange(0, song_length - min_window_length_sec, window_stride_sec):
                     end_sec = start_sec + window_length_sec
 

tokenization_cm3p.py

@@ -295,7 +295,7 @@ class CM3PBeatmapTokenizer(PreTrainedTokenizer):
         if not save_directory:
             raise ValueError("The save_directory must be specified.")
 
-        vocab_file = f"{save_directory}/{filename_prefix or ""}vocab.json"
+        vocab_file = f"{save_directory}/{filename_prefix or ''}vocab.json"
         with open(vocab_file, 'w', encoding='utf-8') as f:
             json.dump(self.vocab, f, ensure_ascii=False)
 
@@ -796,7 +796,7 @@ class CM3PMetadataTokenizer(PreTrainedTokenizer):
         if not save_directory:
             raise ValueError("The save_directory must be specified.")
 
-        vocab_file = f"{save_directory}/{filename_prefix or ""}vocab.json"
+        vocab_file = f"{save_directory}/{filename_prefix or ''}vocab.json"
         with open(vocab_file, 'w', encoding='utf-8') as f:
             json.dump(self.vocab, f, ensure_ascii=False)