#================================================================================= # https://huggingface.co/spaces/projectlosangeles/Orpheus-Masked-Pitches-Inpainter #================================================================================= print('=' * 70) print('Orpheus Masked Pitches Inpainter Gradio App') print('=' * 70) import os os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" os.environ['USE_FLASH_ATTENTION'] = '1' import time as reqtime from pytz import timezone import torch torch.set_float32_matmul_precision('high') torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.allow_tf32 = True torch.backends.cuda.enable_mem_efficient_sdp(True) torch.backends.cuda.enable_math_sdp(True) torch.backends.cuda.enable_flash_sdp(True) torch.backends.cuda.enable_cudnn_sdp(True) import spaces import gradio as gr from x_transformer_2_3_1 import * import datetime import random import tqdm from midi_to_colab_audio import midi_to_colab_audio import TMIDIX import matplotlib.pyplot as plt from huggingface_hub import hf_hub_download # ================================================================================================= OUTPUT_MIDIS_DIR = 'output_midis' # ================================================================================================= print('=' * 70) print('Loading models...') print('=' * 70) print('Loading Orpheus masked encoder model...') print('=' * 70) SEQ_LEN = 2048 PAD_IDX = 18820 DEVICE = 'cuda' model = TransformerWrapper( num_tokens = PAD_IDX+1, max_seq_len = SEQ_LEN, attn_layers = Encoder(dim = 2048, depth = 12, heads = 16, rotary_pos_emb = True, attn_flash = True ) ) model.to(DEVICE) print('=' * 70) print('Loading model checkpoint...') checkpoint = hf_hub_download( repo_id='asigalov61/Orpheus-Music-Transformer', filename='Orpheus_Music_Transformer_Masked_Encoder_Trained_Model_23000_steps_0.6548_loss_0.8132_acc.pth' ) model.load_state_dict(torch.load(checkpoint, map_location=DEVICE, weights_only=True)) model.eval() # model = torch.compile(model) print('=' * 70) print('Done!') print('=' * 70) # ================================================================================================= dtype = torch.bfloat16 ctx = torch.amp.autocast(device_type=DEVICE, dtype=dtype) print('Done!') print('=' * 70) # ================================================================================================= print('Loading SoundFont...') SOUNDFONT_PATH = hf_hub_download(repo_id='projectlosangeles/soundfonts4u', repo_type='dataset', filename='SGM-v2.01-YamahaGrand-Guit-Bass-v2.7.sf2' ) print('Done!') print('=' * 70) # ================================================================================================= def load_midi(input_midi): """Process the input MIDI file and create a token sequence.""" raw_score = TMIDIX.midi2single_track_ms_score(input_midi.name, do_not_check_MIDI_signature=True) escore_notes = TMIDIX.advanced_score_processor(raw_score, return_enhanced_score_notes=True, apply_sustain=True ) if escore_notes and escore_notes[0]: escore_notes = TMIDIX.augment_enhanced_score_notes(escore_notes[0], sort_drums_last=True ) escore_notes = TMIDIX.remove_duplicate_pitches_from_escore_notes(escore_notes) escore_notes = TMIDIX.fix_escore_notes_durations(escore_notes, min_notes_gap=0 ) dscore = TMIDIX.delta_score_notes(escore_notes) dcscore = TMIDIX.chordify_score([d[1:] for d in dscore]) melody_chords = [18816] #======================================================= # MAIN PROCESSING CYCLE #======================================================= for i, c in enumerate(dcscore): delta_time = c[0][0] melody_chords.append(delta_time) for e in c: #======================================================= # Durations dur = max(1, min(255, e[1])) # Patches pat = max(0, min(128, e[5])) # Pitches ptc = max(1, min(127, e[3])) # Velocities # Calculating octo-velocity vel = max(8, min(127, e[4])) velocity = round(vel / 15)-1 #======================================================= # FINAL NOTE SEQ #======================================================= # Writing final note pat_ptc = (128 * pat) + ptc dur_vel = (8 * dur) + velocity melody_chords.extend([pat_ptc+256, dur_vel+16768]) return melody_chords else: return [18816] # ================================================================================================= def save_midi(tokens): """Convert token sequence back to a MIDI score and write it using TMIDIX. """ time = 0 dur = 1 vel = 90 pitch = 60 channel = 0 patch = 0 patches = [-1] * 16 channels = [0] * 16 channels[9] = 1 song_f = [] for ss in tokens: if 0 <= ss < 256: time += ss * 16 if 256 <= ss < 16768: patch = (ss-256) // 128 if patch < 128: if patch not in patches: if 0 in channels: cha = channels.index(0) channels[cha] = 1 else: cha = 15 patches[cha] = patch channel = patches.index(patch) else: channel = patches.index(patch) if patch == 128: channel = 9 pitch = (ss-256) % 128 if 16768 <= ss < 18816: dur = ((ss-16768) // 8) * 16 vel = (((ss-16768) % 8)+1) * 15 song_f.append(['note', time, dur, channel, pitch, vel, patch]) if song_f is not None and song_f: song_f = TMIDIX.remove_duplicate_pitches_from_escore_notes(song_f) song_f = TMIDIX.fix_escore_notes_durations(song_f, min_notes_gap=0 ) output_score, patches, overflow_patches = TMIDIX.patch_enhanced_score_notes(song_f) now = datetime.datetime.now(PDT) ms4 = now.strftime("%f")[:4] # first four digits of microseconds fname = ( "Orpheus-Masked-Pitches-Inpainter-Composition-" + now.strftime(f"%Y-%m-%d-%H-%M-%S-{ms4}") ) os.makedirs(OUTPUT_MIDIS_DIR, exist_ok=True) output_fname = os.path.join(OUTPUT_MIDIS_DIR, fname) TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter( output_score, output_signature='Orpheus Masked Pitches Inpainter', output_file_name=output_fname, track_name='Project Los Angeles', list_of_MIDI_patches=patches, verbose=False ) return output_fname, output_score else: return None, None # ================================================================================================= @spaces.GPU def inpaint_pitches(inp_seq, input_patch, input_inpaint_ratio, input_num_prime_notes ): print('*' * 70) print('Inpainting pitches...') inp_seq = inp_seq[:SEQ_LEN] m_pos = [i for i in range(SEQ_LEN) if (128*input_patch)+256 < inp_seq[i] < (128*(input_patch+1))+256] m_pos = m_pos[min(len(m_pos), input_num_prime_notes):] if input_inpaint_ratio < 1: m_pos = sorted(random.sample(m_pos, k=int(round(len(m_pos) * input_inpaint_ratio)))) results = predict_masked_tokens(model, inp_seq, mask_positions=m_pos, topk=1) output_seq = results['predicted_ids'] print('Done!') print('=' * 70) return output_seq # ================================================================================================= def Inpaint_Pitches(input_midi, input_patch, input_inpaint_ratio, input_num_prime_notes ): if input_midi is not None: print('=' * 70) print('Req start time: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now(PDT))) start_time = reqtime.time() print('=' * 70) fn = os.path.basename(input_midi.name) fn1 = fn.split('.')[0] print('Input file name:', fn) print('Input patch:', input_patch) print('Input inpaint ratio:', input_inpaint_ratio) print('Input number of prime notes:', input_num_prime_notes) print('=' * 70) print('Loading MIDI...') inp_seq = load_midi(input_midi) print('Composition has', len(inp_seq), 'tokens') print('Sample composition tokens:', inp_seq[:5]) print('=' * 70) #=============================================================================== output_seq = inpaint_pitches(inp_seq, input_patch, input_inpaint_ratio, input_num_prime_notes ) #=============================================================================== print('Saving MIDI...') print('=' * 70) output_fname, output_score = save_midi(output_seq) #=============================================================================== print('Rendering results...') print('=' * 70) audio = midi_to_colab_audio(output_fname+'.mid', soundfont_path=SOUNDFONT_PATH, sample_rate=16000, output_for_gradio=True ) #======================================================== output_audio = (16000, audio) output_plot = TMIDIX.plot_ms_SONG(output_score, plot_title=os.path.basename(output_fname)+'.mid', return_plt=True ) print('Done!') print('=' * 70) #======================================================== print('-' * 70) print('Req end time: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now(PDT))) print('-' * 70) print('Req execution time:', (reqtime.time() - start_time), 'sec') return output_audio, output_plot, output_fname+'.mid' return None, None, None # ================================================================================================= PDT = timezone('US/Pacific') print('=' * 70) print('App start time: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now(PDT))) print('=' * 70) soundfont = "SGM-v2.01-YamahaGrand-Guit-Bass-v2.7.sf2" app = gr.Blocks() with app: gr.Markdown("