- Replaced the model with another fine-tuned GPT-2 with custom tokens.

.gitignore

@@ -0,0 +1,2 @@
+venv/
+__pycache__/

app.py

@@ -1,24 +1,31 @@
 import gradio as gr
 from beatgenerator import BeatGenerator
 from datetime import datetime
+from transformers import GPT2LMHeadModel, GPT2Tokenizer
 
 STEP_COUNT = 32
-INSTRUMENT_COUNT = 3
+INSTRUMENT_COUNT = 9
 
-def on_submit(*grid_rows) -> str:
+model = GPT2LMHeadModel.from_pretrained("./model")
+tokenizer = GPT2Tokenizer.from_pretrained("./tokenizer")
+tokenizer.pad_token = tokenizer.eos_token
+beat_generator = BeatGenerator(model=model, tokenizer=tokenizer)
+
+def on_submit(*grid_rows) -> [str]:
     step_data_container = []
 
     for grid_row_id in range(INSTRUMENT_COUNT):
         grid_row_as_ints = list(map(lambda x: int(x) - 1, grid_rows[grid_row_id]))
         step_data_container.append(grid_row_as_ints)
 
-    beat_generator = BeatGenerator(data_container=step_data_container, temperature=grid_rows[3])
-    genre, beat_data = beat_generator.make_beat()
-
+    temperature: float = grid_rows[9]
+    tempo: int = grid_rows[10]
     now = datetime.now()
     date_string = now.strftime("%Y-%m-%d_%H-%M")
+    
+    genre, midi_data = beat_generator.generate_beat(user_prompt=step_data_container, temperature=temperature, tempo=tempo)
 
-    return """<div><h3>Genre: {0}</h3></div><br/><div><a href="data:audio/midi;base64,{1}" download="beat-{0}-{2}.mid">Download beat</a></div>""".format(genre, beat_data, date_string)
+    return ["""<div><h3>Genre: {0}</h3></div><br/><div><a href="data:audio/midi;base64,{1}" download="beat-{0}-{2}.mid">Download beat</a></div>""".format(genre, midi_data, date_string)]
 
 checkbox_rows = [
     ["{:02d}".format(col + 1) for col in range(STEP_COUNT)] for _ in range(INSTRUMENT_COUNT)
@@ -27,8 +34,15 @@ checkbox_rows = [
 inputs = [
     gr.inputs.CheckboxGroup(checkbox_rows[0], label=f"Kick"),
     gr.inputs.CheckboxGroup(checkbox_rows[1], label=f"Snare"),
+    gr.inputs.CheckboxGroup(checkbox_rows[2], label=f"Clap"),
     gr.inputs.CheckboxGroup(checkbox_rows[2], label=f"Hat"),
-    gr.inputs.Slider(minimum=0.1, maximum=1.0, step=0.1, default=0.7, label="Temperature")
+    gr.inputs.CheckboxGroup(checkbox_rows[2], label=f"L tom"),
+    gr.inputs.CheckboxGroup(checkbox_rows[2], label=f"Open hat"),
+    gr.inputs.CheckboxGroup(checkbox_rows[2], label=f"M tom"),
+    gr.inputs.CheckboxGroup(checkbox_rows[2], label=f"Crash cymbal"),
+    gr.inputs.CheckboxGroup(checkbox_rows[2], label=f"Ride cymbal"),
+    gr.inputs.Slider(minimum=0.1, maximum=1.0, step=0.1, default=0.7, label="Temperature"),
+    gr.inputs.Slider(minimum=60, maximum=200, step=1, default=120, label="Tempo")
 ]
 
 iface = gr.Interface(
@@ -36,7 +50,7 @@ iface = gr.Interface(
     inputs=inputs,
     outputs=["html"],
     title="Simple (MIDI) Beat Generator",
-    description="A simple beat generator that creates an 8-bar MIDI beats on every run, based on a 32-step (2 bars) prompt in the form of a step sequencer with three instruments: kick, snare and hihat. The generator uses a small fine-tuned GPT-2 model to recognise the genre (currently only Trap and Deep House) and generate the beat. The result is a mixture between the original prompt and the generated material. Higher temperature values may introduce more instruments to the generated beat."
+    description="A simple beat generator that creates an 8-bar MIDI beats on every run, based on a 32-step (2 bars) prompt in the form of a step sequencer. The generator uses a small fine-tuned GPT-2 model to recognise the genre (currently only Trap and Deep House) and generate the beat."
 )
 
 iface.launch()

beatgenerator.py

@@ -2,175 +2,54 @@ from midiutil import MIDIFile
 import base64
 from io import BytesIO
 from transformers import GPT2LMHeadModel, GPT2Tokenizer
-import random
+from customtokenencoderdecoder import CustomTokenEncoderDecoder
 
 class BeatGenerator:
     STEP_SIZE = 0.25
+    STEPS_PER_SEQUENCE = 32
 
-    def __init__(self, data_container: [[int]], temperature: [float], model: str = "gpt2"):
-        self.__data_container = data_container
-        self.__temperature = temperature
-        self.__model = GPT2LMHeadModel.from_pretrained("./model")
-        tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
-        tokenizer.pad_token = tokenizer.eos_token
+    def __init__(self, model: GPT2LMHeadModel, tokenizer: GPT2Tokenizer):
+        self.__model = model
         self.__tokenizer = tokenizer
-        self.__model_type = model
-        self.__promptAsMusicEvents: [[int, int]] = []
+        self.__sections = ["a", "b", "c", "d"]
 
-    # Public methods
-    def make_beat(self) -> [str, str]:
-        genre_prompt = self.__make_genre_prompt()
-        genre = self.__make_genre_inference_gpt_2(prompt=genre_prompt, temperature=self.__temperature)
-        beat_prompt = self.__make_beat_prompt(genre=genre)
-        sequence = self.__make_inference_gpt_2(prompt=beat_prompt, temperature=self.__temperature)
-        music_events_container = self.__make_music_events(sequence=sequence)
-        composite_events = self.__make_composite_events(events=music_events_container)
-
-        midi_buffer = self.__make_midi_buffer(
-            data_container=composite_events,
-            verbose=False
-        )
-        midi_base64 = base64.b64encode(midi_buffer.read()).decode("utf-8")
+    def generate_beat(self, user_prompt: [[int]], temperature: float, tempo: float) -> [str, str]:
+        # pitches = [36, 38, 42]
+        pitches = [36, 38, 39, 42, 45, 46, 47, 49, 51]
+        assert len(user_prompt) == len(pitches), "User prompt length must be equal to the number of pitches"
         
-        return genre, midi_base64
-    
-    # Private methods
-    def __make_genre_prompt(self) -> str:
-        pitches: [int] = [36, 38, 42]
-        prompt_prefix = "Seed "
-        prompt: str = ""
-        prompt_suffix = "Genre"
-
-        for instrument_id, instrument_steps in enumerate(self.__data_container):
-            for step in instrument_steps:
-                prompt += "({0},{1}) ".format(step, pitches[instrument_id])
-                self.__promptAsMusicEvents.append((step, pitches[instrument_id]))
-
-        return prompt_prefix + prompt + prompt_suffix
-    
-    def __make_beat_prompt(self, genre: str) -> str:
-        pitches: [int] = [36, 38, 42]
-        prompt_prefix = "Seed "
-        prompt: str = ""
-        prompt_suffix = genre + " Beat"
-
-        for instrument_id, instrument_steps in enumerate(self.__data_container):
-            for step in instrument_steps:
-                prompt += "({0},{1}) ".format(step, pitches[instrument_id])
-                self.__promptAsMusicEvents.append((step, pitches[instrument_id]))
-
-        return prompt_prefix + prompt + prompt_suffix
-
-    def __make_music_events(self, sequence: [str]) -> [(int, int)]:
-        minimum_event_length = 5
-        result: [(int, int)] = []
-        sequence_list = sequence.split(" ")
-
-        if len(sequence_list) == 0:
-            return result
-
-        for i in range(len(sequence_list)):
-            if len(sequence_list[i]) >= minimum_event_length and sequence_list[i][0] == '(' and sequence_list[i][-1] == ')':
-                step_pitch = sequence_list[i][1:-1].split(",")
-
-                if len(step_pitch) == 2:
-                    isValid = True
-
-                    for item in step_pitch:
-                        if not item.isdigit():
-                            isValid = False
-                            break
-
-                    if isValid:
-                        result.append((int(step_pitch[0]), int(step_pitch[1])))
-
-        return result
-
-    def __make_inference(self, prompt: str, temperature: float) -> str:
-        if self.__model_type == "gpt2":
-            return self.__make_inference_gpt_2(prompt=prompt, temperature=temperature)
-        else:
-            raise Exception("Invalid model")
-
-    def __make_genre_inference_gpt_2(self, prompt: str, temperature: float) -> str:
-        print("Generating GPT-2 sequence... with temperature: {0}".format(temperature))
-        tokenised_prompt = self.__tokenizer.encode(prompt, add_special_tokens=True, return_tensors="pt")
-        generated_sequence = self.__model.generate(
-            tokenised_prompt,
-            max_length=1024,
-            do_sample=True,
-            temperature=0.5, #temperature,
-            num_return_sequences=1,
-        )
+        user_events: [[int, int]] = []
+        for pitch_id, pitch in enumerate(pitches):
+            for step in user_prompt[pitch_id]: 
+                user_events.append((step, pitch))
         
-        result = self.__tokenizer.decode(
-            generated_sequence[0], skip_special_tokens=True
-        )
-
-        result_as_list = result.split(" ")
-    
-        return self.__find_next_element("Genre", result_as_list)
-
-    def __make_inference_gpt_2(self, prompt: str, temperature: float) -> str:
-        print("Generating GPT-2 sequence... with temperature: {0}".format(temperature))
-        tokenised_prompt = self.__tokenizer.encode(prompt, add_special_tokens=True, return_tensors="pt")
-        generated_sequence = self.__model.generate(
-            tokenised_prompt,
-            max_length=1024,
-            do_sample=True,
-            temperature=temperature,
-            num_return_sequences=1,
+        custom_token_encoder_decoder = CustomTokenEncoderDecoder(
+            events=user_events,
+            sections=self.__sections,
+            steps_per_section=self.STEPS_PER_SEQUENCE,
+            model=self.__model, 
+            tokenizer=self.__tokenizer,
         )
         
-        result = self.__tokenizer.decode(
-            generated_sequence[0], skip_special_tokens=True
-        )
-
-        return result
-    
-    def __matches_probability(self, p: float) -> bool:
-        if p < 0 or p > 1:
-            raise ValueError("Probability should be a number between 0 and 1.")
-        random_number = random.random()
-
-        return random_number <= p
-    
-    def __make_composite_events(self, events: [(int, int)]) -> [(int, int)]:
-        result: [(int, int)] = []
-        kick_probability = 0.95
-        snare_probability = 0.85
-        hihat_probability = 0.7
-        steps_per_time = 32
-        user_events_for_the_whole_beat: [(int, int)] = []
+        result = custom_token_encoder_decoder.generate_events(temperature=temperature)
         
-        for user_event in self.__promptAsMusicEvents:
-            for times in range(4):    
-                user_events_for_the_whole_beat.append((user_event[0] + times * steps_per_time, user_event[1]))
+        genre = result["genre"]
+        events = result["events"]
         
-        unique_user_events = list(filter(lambda x: x not in events, user_events_for_the_whole_beat))
-                
-        for unique_user_event in unique_user_events:
-            probability = 0
-            if unique_user_event[1] == 36:
-                probability = kick_probability
-            elif unique_user_event[1] == 38:
-                probability = snare_probability
-            elif unique_user_event[1] == 42:
-                probability = hihat_probability
-                 
-            if self.__matches_probability(p=probability) == True:
-                result.append(unique_user_event)
+        midi_buffer = self.__make_midi_buffer(
+            data_container=events,
+            tempo=tempo,
+            verbose=False
+        )
+        midi_base64 = base64.b64encode(midi_buffer.read()).decode("utf-8")
         
-        for event in events:
-            result.append(event)
-    
-        return result
+        return genre, midi_base64
     
-    def __make_midi_buffer(self, data_container: [(int, int)], verbose: bool = False) -> BytesIO:
+    def __make_midi_buffer(self, data_container: [(int, int)], tempo: int, verbose: bool = False) -> BytesIO:
         track_count = 1
         out_midi_file = MIDIFile(1)
-        out_midi_file.addTempo(0, 0, 120)
-        
+        out_midi_file.addTempo(0, 0, tempo)
+    
         for data in data_container:
             step = data[0]
             pitch = data[1]
@@ -188,18 +67,15 @@ class BeatGenerator:
                     channel=9, 
                     pitch=pitch, 
                     time=start_time,                         
-                    duration=0.25, 
+                    duration=self.STEP_SIZE, 
                     volume=volume
                 )
 
         buffer = BytesIO()
         out_midi_file.writeFile(buffer)
         buffer.seek(0)
+        
+        with open("out.mid", "wb") as output_file:
+            out_midi_file.writeFile(output_file)
 
-        return buffer
-    
-    def __find_next_element(self, target, string_list) -> str:
-        for i, string in enumerate(string_list[:-1]):
-            if string == target:
-                return string_list[i + 1]
-        return ""  # If the target string is not found or is the last element in the list
+        return buffer

customtokenencoderdecoder.py

@@ -0,0 +1,203 @@
+from transformers import GPT2Tokenizer, GPT2LMHeadModel
+
+class CustomTokenEncoderDecoder:
+    CUSTOM_CLASSIFICATION_TOKEN = "which_genre_section"
+
+    def __init__(self, events: [[int, int]], sections: [str], steps_per_section: int, model: GPT2LMHeadModel, tokenizer: GPT2Tokenizer):
+        self.__model = model
+        self.__tokenizer = tokenizer
+        self.__events = events
+        self.__steps_per_section = steps_per_section
+        self.__sections = sections
+        self.__events_tokens = self.events_to_tokens(events)
+        
+    def is_step_token(self, token: str) -> bool:
+        return token.startswith("step:")
+
+    def is_pitch_token(self, token: str) -> bool:
+        return token.startswith("pitch:")
+
+    def is_genre_token(self, token: str) -> bool:
+        return token.startswith("genre:")
+
+    def is_section_token(self, token: str) -> bool:
+        return token.startswith("section:")
+
+    def token_to_pitch(self, token: str) -> int:
+        return int(token.split(":")[1])
+
+    def token_to_step(self, token: str) -> int:
+        return int(token.split(":")[1])
+
+    def token_to_section(self, token: str) -> str:
+        return token.split(":")[1]
+
+    def token_to_genre(self, token: str) -> str:
+        return token.split(":")[1]
+
+    def pitch_to_token(self, pitch: int) -> str:
+        return "pitch:{0}".format(pitch)
+
+    def step_to_token(self, step: int) -> [str]:
+        return "step:{0}".format(step)
+
+    def section_to_token(self, section: str) -> [str]:
+        return "section:{0}".format(section)
+
+    def events_to_tokens(self, events: [[int, int]]) -> [str]:
+        result: [str] = []
+        
+        for step_id in range(self.__steps_per_section):
+            step_data = list(filter(lambda x: x[0] == step_id, events))
+            
+            if len(step_data) > 0:
+                result.append(self.step_to_token(step_id))
+                step_tokens = list(map(lambda x: self.pitch_to_token(x[1]), step_data))
+                if len(step_tokens) > 0:
+                    result += step_tokens
+        
+        return result
+
+    def tokens_to_classification_prompt(self, tokens: [str]) -> str:
+        return " ".join(tokens + [self.CUSTOM_CLASSIFICATION_TOKEN])
+    
+    def tokens_to_section_prompt(self, tokens: [str], section: str, prompted_section: str) -> str:
+        return " ".join([self.section_to_token(section)] + tokens + [self.section_to_token(prompted_section)])
+
+    def tokens_to_genre_section(self, tokens: [str]) -> dict:
+        genre: str = ""
+        section: str = ""
+        
+        for token in tokens:
+            if self.is_genre_token(token):
+                genre = self.token_to_genre(token)
+            elif self.is_section_token(token):
+                section = self.token_to_section(token)
+                
+        return { "genre": genre, "section": section }
+
+    def section_to_step_offset(self, section: str) -> int:
+        if section == "a":
+            return 0
+        elif section == "b":
+            return self.__steps_per_section
+        elif section == "c":
+            return 2 * self.__steps_per_section
+        elif section == "d":
+            return 3 * self.__steps_per_section
+        else:
+            raise Exception("Invalid section: {0}".format(section))
+        
+    def tokens_to_section_events(self, tokens: [str], section: str, step_offset: int = None) -> [[int, int]]:
+        for (token_id, token) in enumerate(tokens):
+            if self.is_section_token(token):
+                if self.token_to_section(token) == section:
+                    offset: int = self.section_to_step_offset(section)
+                    if step_offset is not None:
+                        offset = step_offset
+                    return self.tokens_to_events(tokens=tokens[token_id:], step_offset=offset)
+        
+        raise Exception("Section {0} not found in tokens".format(section))
+        
+    def tokens_to_events(self, tokens: [str], step_offset: int) -> [[int, int]]:
+        result: [[int, int]] = []
+        
+        for (token_id, token) in enumerate(tokens):
+            if self.is_step_token(token):
+                step = self.token_to_step(token) + step_offset
+                next_token_id = token_id + 1
+                
+                while next_token_id < len(tokens) and self.is_pitch_token(tokens[next_token_id]):
+                    pitch = self.token_to_pitch(tokens[next_token_id])
+                    result.append((step, pitch))
+                    next_token_id += 1
+                
+        return result
+
+    def convert_events_to_section_events(self, events: [[int, int]], section: str) -> [[int, int]]:
+        offset = self.step_offset_for_section(section)
+        return list(map(lambda x: (x[0] + offset, x[1]), events))
+    
+    def generate_events(self, temperature: float) -> dict:
+        genre_section_data = self.make_classification_inference(temperature=temperature)
+        genre = genre_section_data["genre"]
+        section = genre_section_data["section"]
+        print("Classification results")
+        print("======================")
+        print("Found genre: {0}".format(genre))
+        print("Found section: {0}".format(section))
+        print("======================")
+        
+        all_events: [[int, int]] = []
+        
+        all_events += list(map(lambda x: (x[0] + self.section_to_step_offset(section=section), x[1]) ,self.__events))
+        
+        if section not in self.__sections:
+            raise Exception("Section {0} not found in sections".format(section))
+        
+        other_sections = list(filter(lambda x: x != section, self.__sections))
+        for other_section in other_sections:
+            prompt = self.tokens_to_section_prompt(tokens=self.__events_tokens, section=section, prompted_section=other_section)
+            events = self.make_section_events_inference(prompt=prompt, temperature=temperature, section=other_section, known_section=section)
+            all_events += events    
+       
+        return {
+            "events": all_events,
+            "genre": genre       
+        }
+        
+    def tokens_to_genre_and_section_information(self, tokens: [str]) -> dict:
+        genre: str = ""
+        section: str = ""
+        
+        for token in tokens:
+            if self.is_genre_token(token):
+                genre = self.token_to_genre(token)
+            elif self.is_section_token(token):
+                section = self.token_to_section(token)
+                
+
+        return { "genre": genre, "section": section }
+        
+    def make_classification_inference(self, temperature: float) -> dict:
+        genre_and_section_prompt = self.tokens_to_classification_prompt(self.__events_tokens)
+        prompt = self.__tokenizer.encode(genre_and_section_prompt, add_special_tokens=True, return_tensors="pt")
+        
+        generated_section_genre_sequence = self.__model.generate(
+            prompt,
+            max_length=1024,
+            do_sample=True,
+            temperature=0.1,
+            num_return_sequences=1,
+        )
+        
+        section_genre_result = self.__tokenizer.decode(generated_section_genre_sequence[0], skip_special_tokens=True)
+        assert len(section_genre_result) > 0, "Empty result"
+        
+        genre_section_data = self.tokens_to_genre_and_section_information(section_genre_result.split(" "))
+        return genre_section_data
+    
+    def make_section_events_inference(self, prompt: str, section: str, temperature: float, known_section: str) -> [[int, int]]:
+        tokenised_prompt = self.__tokenizer.encode(prompt, add_special_tokens=True, return_tensors="pt")
+        assert len(tokenised_prompt[0]) <= 1024, "Prompt length exceeds maximum sequence length"
+    
+        generated_sequence = self.__model.generate(
+                tokenised_prompt,
+                max_length=1024,
+                do_sample=True,
+                temperature=temperature,
+                num_return_sequences=1,
+        )
+        
+        result = self.__tokenizer.decode(
+                generated_sequence[0], skip_special_tokens=True
+        )
+
+        events = self.tokens_to_section_events(tokens=result.split(" "), section=section)
+        # Fallback option when inference fails (sometimes the model generates a sequence that doesn't contain the section)
+        if len(events) == 0:
+            events = self.tokens_to_section_events(tokens=result.split(" "), section=known_section, step_offset=self.section_to_step_offset(section=section))
+            
+        assert len(events) > 0, "Empty result"
+        
+        return events

model/config.json

@@ -33,7 +33,7 @@
     }
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.27.4",
+  "transformers_version": "4.28.1",
   "use_cache": true,
-  "vocab_size": 50257
+  "vocab_size": 50321
 }

model/generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 50256,
   "eos_token_id": 50256,
-  "transformers_version": "4.27.4"
+  "transformers_version": "4.28.1"
 }

model/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9fa41174f8226daad7dca89c28f3b94ffc20b668dd85b4450d65f43bb07a23a9
-size 510398013
+oid sha256:f1e51cee355f39d25c000d17d50c4313dd1787f086ce23191b8a495f9c33a82b
+size 510594621

model/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4126f3661406a7f858e5e8758fd29a8517d37c711dffa94648b6153e41ed1659
-size 3643
+oid sha256:94b82dda5c87ea468fb088c62a5c04ce9158aed8f35b34ed6e7ab193f4cb4c8f
+size 3707

requirements.txt

@@ -1,3 +1,4 @@
+gradio
 MIDIUtil
 transformers
-torch
+torch

tokenizer/added_tokens.json

@@ -0,0 +1,66 @@
+{
+  "\n\n###\n\n": 50320,
+  "genre:DHouse": 50314,
+  "genre:Trap": 50313,
+  "pitch:36": 50289,
+  "pitch:37": 50290,
+  "pitch:38": 50291,
+  "pitch:39": 50292,
+  "pitch:40": 50293,
+  "pitch:41": 50294,
+  "pitch:42": 50295,
+  "pitch:43": 50296,
+  "pitch:44": 50297,
+  "pitch:45": 50298,
+  "pitch:46": 50299,
+  "pitch:47": 50300,
+  "pitch:48": 50301,
+  "pitch:49": 50302,
+  "pitch:50": 50303,
+  "pitch:51": 50304,
+  "pitch:52": 50305,
+  "pitch:53": 50306,
+  "pitch:54": 50307,
+  "pitch:55": 50308,
+  "pitch:56": 50309,
+  "pitch:57": 50310,
+  "pitch:58": 50311,
+  "pitch:59": 50312,
+  "section:a": 50315,
+  "section:b": 50316,
+  "section:c": 50317,
+  "section:d": 50318,
+  "step:0": 50257,
+  "step:1": 50258,
+  "step:10": 50267,
+  "step:11": 50268,
+  "step:12": 50269,
+  "step:13": 50270,
+  "step:14": 50271,
+  "step:15": 50272,
+  "step:16": 50273,
+  "step:17": 50274,
+  "step:18": 50275,
+  "step:19": 50276,
+  "step:2": 50259,
+  "step:20": 50277,
+  "step:21": 50278,
+  "step:22": 50279,
+  "step:23": 50280,
+  "step:24": 50281,
+  "step:25": 50282,
+  "step:26": 50283,
+  "step:27": 50284,
+  "step:28": 50285,
+  "step:29": 50286,
+  "step:3": 50260,
+  "step:30": 50287,
+  "step:31": 50288,
+  "step:4": 50261,
+  "step:5": 50262,
+  "step:6": 50263,
+  "step:7": 50264,
+  "step:8": 50265,
+  "step:9": 50266,
+  "which_genre_section": 50319
+}

tokenizer/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer/tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 1024,
+  "pad_token": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}