Upload 6 files

README.md

@@ -1,14 +1,17 @@
 ---
-title: MAI MidiAI Playback
-emoji: 📚
-colorFrom: pink
-colorTo: green
+title: MAI MIDI AI Music Generation Model
+Author: Tara Manuel
+colorFrom: purple
+colorTo: gray
 sdk: gradio
-sdk_version: 5.7.1
+sdk_version: 4.43.0
 app_file: app.py
 pinned: false
 license: mit
-short_description: 'MAI: MIDI AI Music Generation Model'
+
 ---
+This project implements a GPT-2 based model for generating MIDI music sequences
+using deep learning. The model is trained on MIDI files from the MAESTRO
+dataset, converted to a special JSON format. The model uses HuggingFace's
+transformers library for training. 
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

genprocessor.py

@@ -0,0 +1,253 @@
+# -*- coding: utf-8 -*-
+"""genprocessor.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/1kvkhcC2RFcAMNh-jOb6NLFtX_lftKi3I
+"""
+
+#Process generated text to MIDI compatable file
+import re
+from typing import Dict
+import mido
+
+class GENProcessor:
+    def __init__(self):
+        self.START_TRACK = "<|START_TRACK|>"
+        self.END_TRACK = "<|END_TRACK|>"
+        self.START_METADATA = "<|START_METADATA|>"
+        self.END_METADATA = "<|END_METADATA|>"
+
+        self.field_order = {
+            "metadata": ["type", "ticks_per_beat"],
+            "tempo": ["type", "time", "tempo"],
+            "time_signature": ["type", "time", "numerator", "denominator"],
+            "track_name": ["type", "time", "name"],
+            "program_change": ["type", "time", "channel", "program"],
+            "control_change": ["type", "time", "channel", "control", "value"],
+            "note_on": ["type", "time", "channel", "note", "velocity"]
+        }
+
+    def sanitize_event(self, event):
+        """make sure events have all required fields"""
+        if not event or 'type' not in event:
+            return None
+
+        event_type = event['type']
+        required_fields = {
+            'note_on': {'time', 'channel', 'note', 'velocity'},
+            'note_off': {'time', 'channel', 'note', 'velocity'},
+            'control_change': {'time', 'channel', 'control', 'value'},
+            'program_change': {'time', 'program'},
+            'time_signature': {'time', 'numerator', 'denominator'}
+        }
+
+        if event_type in required_fields:
+            missing_fields = required_fields[event_type] - set(event.keys())
+
+            if missing_fields:
+                if event_type == 'time_signature':
+                    event['numerator'] = 4  # set default
+                    event['denominator'] = 4  # set default
+                    event['time'] = event.get('time', 0)
+                else:
+                    return None
+
+            try:
+                # validate fields
+                if 'time' in event:
+                    event['time'] = max(0, int(event['time']))
+                if 'channel' in event:
+                    event['channel'] = max(0, int(event['channel']))
+                if 'note' in event:
+                    event['note'] = max(0, int(event['note']))
+                if 'velocity' in event:
+                    event['velocity'] = max(0, int(event['velocity']))
+                if 'control' in event:
+                    event['control'] = max(0, int(event['control']))
+                if 'value' in event:
+                    event['value'] = max(0, int(event['value']))
+                if 'program' in event:
+                    event['program'] = max(0, int(event['program']))
+                if 'numerator' in event:
+                    numerator = int(event['numerator'])
+                    event['numerator'] = min(4, max(2, numerator))
+                if 'denominator' in event:
+                    event['denominator'] = 4
+
+            except (ValueError, TypeError):
+                return None
+
+        return event
+
+
+    def parse_event_params(self, text: str) -> Dict:
+        """Parse parameters from a line of text."""
+        return {p.split('=', 1)[0].strip(): p.split('=', 1)[1].strip()
+          for p in text.split() if '=' in p and len(p.split('=', 1)) == 2}
+
+    def decode_midi_file(self, text: str) -> Dict:
+        """Decode text representation of a MIDI file into dictionary."""
+        # Create template with defaults in case data is missing
+        midi_data = {
+            "metadata": {
+                "ticks_per_beat": 480
+            },
+            "tracks": [
+                [  # First track always contains tempo and time signature, set defaults
+                    {
+                        "type": "tempo",
+                        "time": 0,
+                        "tempo": 500000
+                    },
+                    {
+                        "type": "time_signature",
+                        "time": 0,
+                        "numerator": 4,
+                        "denominator": 4
+                    }
+                ]
+            ]
+        }
+
+        # Parse the text to get all metadata values
+        metadata_values = {}
+        for line in text.split():
+            if "ticks_per_beat" in line or "ticks_beat" in line:
+                match = re.search(r"ticks[_]?(?:per_)?beat=(\d+)", line)
+                if match:
+                    metadata_values["ticks_per_beat"] = max(75, int(match.group(1)))
+            elif "tempo" in line and "time=0" in line:
+                match = re.search(r"tempo=(\d+)", line)
+                if match:
+                    metadata_values["tempo"] = int(match.group(1))
+
+        # Update template with any found metadata values
+        if "ticks_per_beat" in metadata_values:
+            midi_data["metadata"]["ticks_per_beat"] = metadata_values["ticks_per_beat"]
+        if "tempo" in metadata_values:
+            midi_data["tracks"][0][0]["tempo"] = metadata_values["tempo"]
+
+        # parse the actual events
+        current_track = []
+        building_event = None
+        collecting_params = {}
+
+        for line in text.split():
+            line = line.strip()
+            if not line:
+                continue
+
+            # Skip metadata lines we already processed
+            if "ticks_per_beat" in line or "tempo=0" in line:
+                continue
+
+            # Track boundaries
+            if "<|START_TRACK|>" in line:
+                if len(midi_data["tracks"]) == 1:  # If we're starting the second track
+                    current_track = []
+                continue
+
+            if "<|END_TRACK|>" in line:
+                if current_track:  # Only add non-empty tracks after the first one
+                    midi_data["tracks"].append(current_track)
+                current_track = []
+                building_event = None
+                collecting_params = {}
+                continue
+
+            # Handle events
+            if line.startswith("<") and ">" in line:
+                if building_event and collecting_params:
+                    full_event = {**building_event, **collecting_params}
+                    sanitized = self.sanitize_event(full_event)
+                    if sanitized:
+                        current_track.append(sanitized)
+
+                event_type = re.match(r"<(\w+)>", line)
+                if event_type and event_type.group(1) not in ['START_METADATA', 'composer_', 'position_']:
+                    building_event = {"type": event_type.group(1)}
+                    params_text = line[line.find(">") + 1:].strip()
+                    collecting_params = self.parse_event_params(params_text)
+                    continue
+
+            # Collect additional parameters
+            if building_event and '=' in line:
+                collecting_params.update(self.parse_event_params(line))
+
+        # Add any remaining events in the last track
+        if current_track:
+            midi_data["tracks"].append(current_track)
+
+        return midi_data
+
+def generated_tokens_to_midi(tokens, output_path):
+    """Convert tokenized musical events back into an audio MIDI file."""
+    midi_file = mido.MidiFile(ticks_per_beat=tokens["metadata"]["ticks_per_beat"])
+
+    for track_tokens in tokens["tracks"]:
+        track = mido.MidiTrack()
+        midi_file.tracks.append(track)
+
+        last_time = 0
+
+        # sort events by time
+        sorted_tokens = sorted(track_tokens, key=lambda x: x["time"])
+
+        for token in sorted_tokens:
+            # Calculate time
+            delta_time = token["time"] - last_time
+            last_time = token["time"]
+
+            if token["type"] == "note_on":
+                msg = mido.Message('note_on',
+                                 channel=token["channel"],
+                                 note=token["note"],
+                                 velocity=token["velocity"],
+                                 time=int(delta_time))
+                track.append(msg)
+
+            elif token["type"] == "note_off":
+                msg = mido.Message('note_off',
+                                 channel=token["channel"],
+                                 note=token["note"],
+                                 velocity=token["velocity"],
+                                 time=int(delta_time))
+                track.append(msg)
+
+            elif token["type"] == "program_change":
+                msg = mido.Message('program_change',
+                                 channel=token["channel"],
+                                 program=token["program"],
+                                 time=int(delta_time))
+                track.append(msg)
+
+            elif token["type"] == "control_change":
+                msg = mido.Message('control_change',
+                                 channel=token["channel"],
+                                 control=token["control"],
+                                 value=token["value"],
+                                 time=int(delta_time))
+                track.append(msg)
+
+            elif token["type"] == "tempo":
+                msg = mido.MetaMessage('set_tempo',
+                                     tempo=token["tempo"],
+                                     time=int(delta_time))
+                track.append(msg)
+
+            elif token["type"] == "time_signature":
+                msg = mido.MetaMessage('time_signature',
+                                     numerator=token["numerator"],
+                                     denominator=token["denominator"],
+                                     time=int(delta_time))
+                track.append(msg)
+
+            elif token["type"] == "track_name":
+                msg = mido.MetaMessage('track_name',
+                                     name=token["name"],
+                                     time=int(delta_time))
+                track.append(msg)
+
+    midi_file.save(output_path)

midimusicgenapp.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+"""MidiMusicGenApp.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/1Dn99ii_FiQTx-z5B0dX0br0Gc0U9MUqD
+"""
+
+import gradio as gr
+import torch
+from transformers import GPT2LMHeadModel
+from miditokenizer import MIDITokenizer
+from genprocessor import GENProcessor, generated_tokens_to_midi
+from midi2audio import FluidSynth
+from pydub import AudioSegment
+import tempfile
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+# Load model and tokenizer
+
+torch.serialization.add_safe_globals([set])
+torch.serialization.add_safe_globals([GPT2LMHeadModel])
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = torch.load('model_complete_18epochs.pkl',map_location=device, weights_only=False)
+tokenizer = MIDITokenizer()
+processor = GENProcessor()
+model.eval()
+
+#functions to adjust timing & combine generated song parts
+def adjust_midi_timing(midi_data, start_time=0):
+   """Adjust MIDI timing with optional start time. Prevent large gaps based on ticks_per_beat."""
+   try:
+       # Keep tempo track separate
+       tempo_track = midi_data['tracks'][0]
+       ticks_per_beat = midi_data['metadata']['ticks_per_beat']
+
+       # Calculate thresholds based on ticks_per_beat
+       gap_threshold = ticks_per_beat * 2
+       small_increment = ticks_per_beat // 8  # Eighth note
+
+       # Get all other events and sort by time
+       all_events = []
+       for track in midi_data['tracks'][1:]:
+           all_events.extend(track)
+       all_events.sort(key=lambda x: x['time'])
+
+       # Find sequential times, ignoring large gaps
+       sequential_events = []
+       current_time = all_events[0]['time'] if all_events else 0
+
+       for event in all_events:
+           if event['time'] - current_time > gap_threshold:
+               event['time'] = current_time + small_increment
+           current_time = event['time']
+           sequential_events.append(event)
+
+       # Find first non-zero time
+       first_time = min((event['time'] for event in sequential_events if event['time'] != 0), default=0)
+
+       adjusted_data = {'metadata': midi_data['metadata'], 'tracks': [tempo_track]}
+
+       # Adjust all events
+       adjusted_track = []
+       for event in sequential_events:
+           adjusted_event = event.copy()
+           if event['time'] != 0:
+               adjusted_event['time'] = (event['time'] - first_time) + start_time
+           else:
+               adjusted_event['time'] = start_time
+           adjusted_track.append(adjusted_event)
+
+       adjusted_data['tracks'].append(adjusted_track)
+       return adjusted_data
+
+   except Exception as e:
+       print(f"Error adjusting MIDI timing: {str(e)}")
+       return midi_data
+
+#Functions to generate music
+
+def generate_music(prompt):
+    """Generate music based on a given prompt."""
+    # Tokenize
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+
+    inputs = tokenizer(
+        prompt,
+        return_tensors="pt",
+        padding=True,
+        truncation=True,
+        add_special_tokens=True
+    )
+
+    # Generate
+    output_sequences = model.generate(
+        input_ids=inputs["input_ids"].to(model.device),
+        attention_mask=inputs["attention_mask"].to(model.device),
+        max_length=1024,
+        do_sample=True,
+        temperature=0.6, #adjust creativity
+        top_k=30,
+        top_p=0.90,
+        pad_token_id=tokenizer.eos_token_id,
+        eos_token_id=tokenizer.eos_token_id,
+    )
+
+    # Decode the generated sequence
+    generated_text = tokenizer.decode(output_sequences[0])
+
+    return generated_text
+
+def generate_wrapper(composer):
+    # Format the prompt with the selected composer
+    prompt = f"<|START_METADATA|> <|composer_{composer}|><metadata> ticks_per_beat="
+    generated_text = generate_music(prompt)
+    midi_data=adjust_midi_timing(processor.decode_midi_file(generated_text))
+    print(midi_data)
+    # Create temp file for MIDI
+    with tempfile.NamedTemporaryFile(suffix='.mid', delete=False) as tmp:
+        generated_tokens_to_midi(midi_data, tmp.name)
+        # Convert to WAV
+        fs = FluidSynth(sound_font='FluidR3Mono_GM.sf3')
+        wav_file = tmp.name.replace('.mid', '.wav')
+        fs.midi_to_audio(tmp.name, wav_file)
+
+        # Convert to MP3
+        mp3_file = wav_file.replace('.wav', '.mp3')
+        audio = AudioSegment.from_wav(wav_file)
+        audio.export(mp3_file, format="mp3")
+        return mp3_file
+
+iface = gr.Interface(
+    fn=generate_wrapper,
+    inputs=[
+        gr.Dropdown(
+            choices=["Bach", "Chopin"],
+            label="Select Composer",
+            value="Bach"  # default value
+        )
+    ],
+    outputs=gr.Audio(type="filepath",label="Generated MIDI"),
+    title="MAI: MIDI AI Music Generation Model",
+    description="Select a composer whose musical style you'd like to emulate. Generate an original sequence inspired by that composer's unique sound. It should take a few minutes. Once it's ready, you can listen to the clip or download the audio file."
+    #description="Compose Music in the Style of Your Favorite Composer. Select a composer to generate a music sequence in the style of selected composer",
+    #flagging_mode="never"
+)
+
+iface.launch()

miditokenizer.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+"""miditokenizer.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/16YJUBYcqKYPVIhwzKNi4ELnTftr2TcUY
+"""
+
+#We use a base GPT2 tokenizer with additional functions to handle composer tokens
+#Datasets are created by processing our files in chunks, due to model sequence limits
+#Position information is added to each chunk as additional pattern/data for training
+
+from transformers import GPT2TokenizerFast, GPT2LMHeadModel
+from torch.utils.data import Dataset
+from pathlib import Path
+import torch
+
+class MIDITokenizer:
+    """tokenization specific to MIDI data with special tokens"""
+    def __init__(self, pretrained_model='gpt2'):
+        self.base_tokenizer = GPT2TokenizerFast.from_pretrained(pretrained_model)
+        special_tokens = {
+            'additional_special_tokens': [
+                '<|START_METADATA|>',
+                '<|END_METADATA|>',
+                '<|START_TRACK|>',
+                '<|END_TRACK|>',
+                '<metadata>',
+                '<tempo>',
+                '<time_signature>',
+                '<program_change>',
+                '<note_on>',
+                '<note_off>',
+                '<control_change>'
+            ],
+            'pad_token': '[PAD]'
+        }
+        self.base_tokenizer.add_special_tokens(special_tokens)
+        self.pad_token_id = self.base_tokenizer.pad_token_id
+        self.eos_token_id = self.base_tokenizer.eos_token_id
+        self.bos_token_id = self.base_tokenizer.bos_token_id
+        self.pad_token = self.base_tokenizer.pad_token
+        self.eos_token = self.base_tokenizer.eos_token
+        self.bos_token = self.base_tokenizer.bos_token
+
+
+    def add_composer_tokens(self, composers):
+        #composer tokens
+        composer_tokens = [f'<|composer_{c}|>' for c in composers]
+        self.base_tokenizer.add_special_tokens({
+            'additional_special_tokens': composer_tokens
+        })
+
+    def __call__(self, text, **kwargs):
+        return self.base_tokenizer(text, **kwargs)
+
+    def decode(self, token_ids, **kwargs):
+        """Decode while preserving special tokens"""
+        return self.base_tokenizer.decode(token_ids, skip_special_tokens=False, **kwargs)
+
+    def pad(self, *args, **kwargs):
+        return self.base_tokenizer.pad(*args, **kwargs)
+
+    def get_vocab(self):
+        return self.base_tokenizer.get_vocab()

packages.txt

@@ -0,0 +1,2 @@
+fluidsynth
+ffmpeg

requirements.txt

@@ -0,0 +1,6 @@
+gradio
+torch
+transformers
+mido
+midi2audio
+pydub