Delete tokenizer.py

tokenizer.py

@@ -1,33 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep 13 19:15:23 2024
-
-@author: salikha4
-"""
-
-from transformers import PreTrainedTokenizer
-from input_preprocess import gen_tokens
-
-class WirelessChannelTokenizer(PreTrainedTokenizer):
-    """
-    A Hugging Face-compatible tokenizer for wireless channels. 
-    It performs segmentation and masking for wireless channel data.
-    """
-    def __init__(self, patch_size=16, max_len=129, **kwargs):
-        super().__init__(**kwargs)
-        self.patch_size = patch_size
-        self.max_len = max_len
-
-    def preprocess_channels(self, scenario_idxs):
-        # Call gen_tokens() for preprocessing the wireless channel data
-        preprocessed_data, sequence_length, element_length = gen_tokens(
-            scenario_idxs, patch_gen=True, patch_size=self.patch_size,
-            gen_deepMIMO_data=True, gen_raw=True, save_data=False
-        )
-        return preprocessed_data
-
-    def __call__(self, scenario_idxs):
-        return self.preprocess_channels(scenario_idxs)
-
-    def save_pretrained(self, save_directory):
-        super().save_pretrained(save_directory)