los_angeles_music_composer.py

# -*- coding: utf-8 -*-
"""Los_Angeles_Music_Composer.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/github/asigalov61/Los-Angeles-Music-Composer/blob/main/Los_Angeles_Music_Composer.ipynb

# Los Angeles Music Composer (ver. 4.5)

***

Powered by tegridy-tools: https://github.com/asigalov61/tegridy-tools

***

WARNING: This complete implementation is a functioning model of the Artificial Intelligence. Please excercise great humility, care, and respect. https://www.nscai.gov/

***

#### Project Los Angeles

#### Tegridy Code 2023

***

# (GPU CHECK)
"""

#@title NVIDIA GPU check
!nvidia-smi

"""# (SETUP ENVIRONMENT)"""

#@title Install dependencies
!git clone --depth 1 https://github.com/asigalov61/Los-Angeles-Music-Composer
!pip install torch
!pip install einops
!pip install torch-summary
!pip install tqdm
!pip install matplotlib
!apt install fluidsynth #Pip does not work for some reason. Only apt works
!pip install midi2audio

# Commented out IPython magic to ensure Python compatibility.
#@title Import modules

print('=' * 70)
print('Loading core Los Angeles Music Composer modules...')

import os
import pickle
import random
import secrets
import statistics
from time import time
import tqdm

print('=' * 70)
print('Loading main Los Angeles Music Composer modules...')
import torch

# %cd /content/Los-Angeles-Music-Composer

import TMIDIX
from lwa_transformer import *

# %cd /content/
print('=' * 70)
print('Loading aux Los Angeles Music Composer modeules...')

import matplotlib.pyplot as plt

from torchsummary import summary
from sklearn import metrics

from midi2audio import FluidSynth
from IPython.display import Audio, display

from google.colab import files

print('=' * 70)
print('Done!')
print('Enjoy! :)')
print('=' * 70)

"""# (LOAD MODEL)"""

# Commented out IPython magic to ensure Python compatibility.
#@title Unzip Pre-Trained Los Angeles Music Composer Model
print('=' * 70)
# %cd /content/Los-Angeles-Music-Composer/Model

print('=' * 70)
print('Unzipping pre-trained Los Angeles Music Composer model...Please wait...')

!cat /content/Los-Angeles-Music-Composer/Model/Los_Angeles_Music_Composer_Trained_Model.zip* > /content/Los-Angeles-Music-Composer/Model/Los_Angeles_Music_Composer_Trained_Model.zip
print('=' * 70)

!unzip -j /content/Los-Angeles-Music-Composer/Model/Los_Angeles_Music_Composer_Trained_Model.zip
print('=' * 70)

print('Done! Enjoy! :)')
print('=' * 70)
# %cd /content/
print('=' * 70)

#@title Load Los Angeles Music Composer Model
full_path_to_model_checkpoint = "/content/Los-Angeles-Music-Composer/Model/Los_Angeles_Music_Composer_Model_88835_steps_0.643_loss.pth" #@param {type:"string"}

#@markdown Model precision option

model_precision = "bfloat16" # @param ["bfloat16", "float16", "float32"]

#@markdown bfloat16 == Third precision/triple speed (if supported, otherwise the model will default to float16)

#@markdown float16 == Half precision/double speed

#@markdown float32 == Full precision/normal speed

plot_tokens_embeddings = False # @param {type:"boolean"}

print('=' * 70)
print('Loading Los Angeles Music Composer Pre-Trained Model...')
print('Please wait...')
print('=' * 70)
print('Instantiating model...')

torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul
torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn
device_type = 'cuda'

if model_precision == 'bfloat16' and torch.cuda.is_bf16_supported():
  dtype = 'bfloat16'
else:
  dtype = 'float16'

if model_precision == 'float16':
  dtype = 'float16'

if model_precision == 'float32':
  dtype = 'float32'

ptdtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype]
ctx = torch.amp.autocast(device_type=device_type, dtype=ptdtype)

SEQ_LEN = 4096

# instantiate the model

model = LocalTransformer(
    num_tokens = 2831,
    dim = 1024,
    depth = 36,
    causal = True,
    local_attn_window_size = 512,
    max_seq_len = SEQ_LEN
)

model = torch.nn.DataParallel(model)

model.cuda()
print('=' * 70)

print('Loading model checkpoint...')

model.load_state_dict(torch.load(full_path_to_model_checkpoint))
print('=' * 70)

model.eval()

print('Done!')
print('=' * 70)

print('Model will use', dtype, 'precision...')
print('=' * 70)

# Model stats
print('Model summary...')
summary(model)

# Plot Token Embeddings

if plot_tokens_embeddings:
  tok_emb = model.module.token_emb.weight.detach().cpu().tolist()

  cos_sim = metrics.pairwise_distances(
    tok_emb, metric='cosine'
  )
  plt.figure(figsize=(7, 7))
  plt.imshow(cos_sim, cmap="inferno", interpolation="nearest")
  im_ratio = cos_sim.shape[0] / cos_sim.shape[1]
  plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)
  plt.xlabel("Position")
  plt.ylabel("Position")
  plt.tight_layout()
  plt.plot()
  plt.savefig("/content/Los-Angeles-Music-Composer-Tokens-Embeddings-Plot.png", bbox_inches="tight")

"""# (GENERATE)

# (IMPROV)
"""

#@title Improv Generation

#@markdown Improv settings

number_of_instruments = 1 #@param {type:"slider", min:1, max:12, step:1}
lead_instrument = "Piano" #@param ["Piano", "Guitar", "Bass", "Violin", "Cello", "Harp", "Trumpet", "Sax", "Flute", "Drums", "Choir", "Organ"]
add_drums = False #@param {type:"boolean"}

#@markdown Generation settings

number_of_tokens_tp_generate = 201 # @param {type:"slider", min:33, max:2045, step:3}
number_of_batches_to_generate = 4 #@param {type:"slider", min:1, max:16, step:1}
temperature = 1 #@param {type:"slider", min:0.1, max:1, step:0.1}

#@markdown Other settings

allow_model_to_stop_generation_if_needed = False #@param {type:"boolean"}
render_MIDI_to_audio = True # @param {type:"boolean"}

print('=' * 70)
print('Los Angeles Music Composer Improv Model Generator')
print('=' * 70)

if allow_model_to_stop_generation_if_needed:
  min_stop_token = 2816
else:
  min_stop_token = 0

if add_drums:
  drumsp = 1
else:
  drumsp = 0

outy = [2816, 2817+drumsp, 2819+((number_of_instruments)-1)]

instruments_list = ["Piano", "Guitar", "Bass", "Violin", "Cello", "Harp", "Trumpet", "Sax", "Flute", 'Drums', "Choir", "Organ"]
lead_instrument_number = instruments_list.index(lead_instrument)


time = 0
dur_vel = (10 * 8) + (7)
cha_ptc = (lead_instrument_number * 128) + 72

outy.extend([time, dur_vel+128, cha_ptc+1152])

print('Selected Improv sequence:')
print(outy)
print('=' * 70)

inp = [outy] * number_of_batches_to_generate

inp = torch.LongTensor(inp).cuda()

with ctx:
  out = model.module.generate(inp,
                        number_of_tokens_tp_generate,
                        temperature=temperature,
                        return_prime=True,
                        min_stop_token=min_stop_token,
                        verbose=True)

out0 = out.tolist()

print('=' * 70)
print('Done!')
print('=' * 70)

#======================================================================

print('Rendering results...')

for i in range(number_of_batches_to_generate):

  print('=' * 70)
  print('Batch #', i)
  print('=' * 70)

  out1 = out0[i]

  print('Sample INTs', out1[:12])
  print('=' * 70)

  if len(out) != 0:

      song = out1
      song_f = []
      tim = 0
      dur = 0
      vel = 0
      pitch = 0
      channel = 0

      son = []
      song1 = []

      for s in song:
        if s >= 128 and s < (12*128)+1152:
          son.append(s)
        else:
          if len(son) == 3:
            song1.append(son)
          son = []
          son.append(s)

      for ss in song1:

        tim += ss[0] * 10

        dur = (((ss[1]-128) // 8)+1) * 20
        vel = (((ss[1]-128) % 8)+1) * 15

        channel = (ss[2]-1152) // 128
        pitch = (ss[2]-1152) % 128

        song_f.append(['note', tim, dur, channel, pitch, vel ])

      detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                                output_signature = 'Los Angeles Music Composer',
                                                                output_file_name = '/content/Los-Angeles-Music-Composer-Music-Composition_'+str(i),
                                                                track_name='Project Los Angeles',
                                                                list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 19, 0, 0, 0, 0]
                                                                )


      print('=' * 70)
      print('Displaying resulting composition...')
      print('=' * 70)

      fname = '/content/Los-Angeles-Music-Composer-Music-Composition_'+str(i)

      x = []
      y =[]
      c = []

      colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']

      for s in song_f:
        x.append(s[1] / 1000)
        y.append(s[4])
        c.append(colors[s[3]])

      if render_MIDI_to_audio:
        FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
        display(Audio(str(fname + '.wav'), rate=16000))

      plt.figure(figsize=(14,5))
      ax=plt.axes(title=fname)
      ax.set_facecolor('black')

      plt.scatter(x,y, c=c)
      plt.xlabel("Time")
      plt.ylabel("Pitch")
      plt.show()

"""# (CUSTOM MIDI)"""

#@title Load Seed MIDI

#@markdown Press play button to to upload your own seed MIDI or to load one of the provided sample seed MIDIs from the dropdown list below

select_seed_MIDI = "Upload your own custom MIDI" #@param ["Upload your own custom MIDI", "Los-Angeles-Music-Composer-Piano-Seed-1", "Los-Angeles-Music-Composer-Piano-Seed-2", "Los-Angeles-Music-Composer-Piano-Seed-3", "Los-Angeles-Music-Composer-Piano-Seed-4", "Los-Angeles-Music-Composer-Piano-Seed-5", "Los-Angeles-Music-Composer-MI-Seed-1", "Los-Angeles-Music-Composer-MI-Seed-2", "Los-Angeles-Music-Composer-MI-Seed-3", "Los-Angeles-Music-Composer-MI-Seed-4", "Los-Angeles-Music-Composer-MI-Seed-5"]
render_MIDI_to_audio = False # @param {type:"boolean"}

print('=' * 70)
print('Los Angeles Music Composer Seed MIDI Loader')
print('=' * 70)

f = ''

if select_seed_MIDI != "Upload your own custom MIDI":
  print('Loading seed MIDI...')
  f = '/content/Los-Angeles-Music-Composer/Seeds/'+select_seed_MIDI+'.mid'
  score = TMIDIX.midi2single_track_ms_score(open(f, 'rb').read(), recalculate_channels=False)

else:
  print('Upload your own custom MIDI...')
  print('=' * 70)
  uploaded_MIDI = files.upload()
  if list(uploaded_MIDI.keys()):
    score = TMIDIX.midi2single_track_ms_score(list(uploaded_MIDI.values())[0], recalculate_channels=False)
    f = list(uploaded_MIDI.keys())[0]

if f != '':

  print('=' * 70)
  print('File:', f)
  print('=' * 70)

  #=======================================================
  # START PROCESSING

  # INSTRUMENTS CONVERSION CYCLE
  events_matrix = []
  melody_chords_f = []
  melody_chords_f1 = []

  itrack = 1

  patches = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]

  patch_map = [
              [0, 1, 2, 3, 4, 5, 6, 7], # Piano
              [24, 25, 26, 27, 28, 29, 30], # Guitar
              [32, 33, 34, 35, 36, 37, 38, 39], # Bass
              [40, 41], # Violin
              [42, 43], # Cello
              [46], # Harp
              [56, 57, 58, 59, 60], # Trumpet
              [64, 65, 66, 67, 68, 69, 70, 71], # Sax
              [72, 73, 74, 75, 76, 77, 78], # Flute
              [-1], # Drums
              [52, 53], # Choir
              [16, 17, 18, 19, 20] # Organ
              ]

  while itrack < len(score):
    for event in score[itrack]:
        if event[0] == 'note' or event[0] == 'patch_change':
            events_matrix.append(event)
    itrack += 1

  events_matrix.sort(key=lambda x: x[1])

  events_matrix1 = []

  for event in events_matrix:
    if event[0] == 'patch_change':
        patches[event[2]] = event[3]

    if event[0] == 'note':
        event.extend([patches[event[3]]])
        once = False

        for p in patch_map:
            if event[6] in p and event[3] != 9: # Except the drums
                event[3] = patch_map.index(p)
                once = True

        if not once and event[3] != 9: # Except the drums
            event[3] = 15 # All other instruments/patches channel
            event[5] = max(80, event[5])

        if event[3] < 12: # We won't write chans 12-16 for now...
            events_matrix1.append(event)

  #=======================================================
  # PRE-PROCESSING

  # checking number of instruments in a composition
  instruments_list_without_drums = list(set([y[3] for y in events_matrix1 if y[3] != 9]))

  if len(events_matrix1) > 0 and len(instruments_list_without_drums) > 0:

    # recalculating timings
    for e in events_matrix1:
        e[1] = int(e[1] / 10) # Max 1 seconds for start-times
        e[2] = int(e[2] / 20) # Max 2 seconds for durations

    # Sorting by pitch, then by start-time
    events_matrix1.sort(key=lambda x: x[4], reverse=True)
    events_matrix1.sort(key=lambda x: x[1])

    #=======================================================
    # FINAL PRE-PROCESSING

    melody_chords = []

    pe = events_matrix1[0]

    for e in events_matrix1:
      if e[1] >= 0 and e[2] >= 0:

        # Cliping all values...
        tim = max(0, min(127, e[1]-pe[1]))
        dur = max(1, min(127, e[2]))
        cha = max(0, min(11, e[3]))
        ptc = max(1, min(127, e[4]))
        vel = max(8, min(127, e[5]))

        velocity = round(vel / 15)

        # Writing final note
        melody_chords.append([tim, dur, cha, ptc, velocity])

        pe = e

  instruments_list = list(set([y[2] for y in melody_chords]))
  num_instr = len(instruments_list)

  #=======================================================
  # FINAL PROCESSING
  #=======================================================

  # Break between compositions / Intro seq

  if 9 in instruments_list:
    drums_present = 2818 # Yes
  else:
    drums_present = 2817 # No

  melody_chords_f.extend([2816, drums_present, 2819+(num_instr-1)])

  #=======================================================

  # Composition control seq
  intro_mode_time = statistics.mode([0] + [y[0] for y in melody_chords if y[2] != 9 and y[0] != 0])
  intro_mode_dur = statistics.mode([y[1] for y in melody_chords if y[2] != 9])
  intro_mode_pitch = statistics.mode([y[3] for y in melody_chords if y[2] != 9])
  intro_mode_velocity = statistics.mode([y[4] for y in melody_chords if y[2] != 9])

  # Instrument value 12 is reserved for composition control seq
  intro_dur_vel = (intro_mode_dur * 8) + (intro_mode_velocity-1)
  intro_cha_ptc = (12 * 128) + intro_mode_pitch

  melody_chords_f.extend([intro_mode_time, intro_dur_vel+128, intro_cha_ptc+1152])

  # TOTAL DICTIONARY SIZE 2831

  #=======================================================
  # MAIN PROCESSING CYCLE
  #=======================================================

  for m in melody_chords:

    # WRITING EACH NOTE HERE
    dur_vel = (m[1] * 8) + (m[4]-1)
    cha_ptc = (m[2] * 128) + m[3]

    melody_chords_f.extend([m[0], dur_vel+128, cha_ptc+1152])
    melody_chords_f1.append([m[0], dur_vel+128, cha_ptc+1152])

  #=======================================================

  song = melody_chords_f
  song_f = []
  tim = 0
  dur = 0
  vel = 0
  pitch = 0
  channel = 0

  son = []
  song1 = []

  for s in song:
    if s >= 128 and s < (12*128)+1152:
      son.append(s)
    else:
      if len(son) == 3:
        song1.append(son)
      son = []
      son.append(s)

  for ss in song1:

    tim += ss[0] * 10

    dur = ((ss[1]-128) // 8) * 20
    vel = (((ss[1]-128) % 8)+1) * 15

    channel = (ss[2]-1152) // 128
    pitch = (ss[2]-1152) % 128

    song_f.append(['note', tim, dur, channel, pitch, vel ])

  detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                            output_signature = 'Los Angeles Music Composer',
                                                            output_file_name = '/content/Los-Angeles-Music-Composer-Seed-Composition',
                                                            track_name='Project Los Angeles',
                                                            list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 19, 0, 0, 0, 0]
                                                            )

  #=======================================================

  print('=' * 70)
  print('Composition stats:')
  print('Composition has', len(melody_chords_f1), 'notes')
  print('Composition has', len(melody_chords_f), 'tokens')
  print('=' * 70)

  print('Displaying resulting composition...')
  print('=' * 70)

  fname = '/content/Los-Angeles-Music-Composer-Seed-Composition'

  x = []
  y =[]
  c = []

  colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']

  for s in song_f:
    x.append(s[1] / 1000)
    y.append(s[4])
    c.append(colors[s[3]])

  if render_MIDI_to_audio:
    FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
    display(Audio(str(fname + '.wav'), rate=16000))

  plt.figure(figsize=(14,5))
  ax=plt.axes(title=fname)
  ax.set_facecolor('black')

  plt.scatter(x,y, c=c)
  plt.xlabel("Time")
  plt.ylabel("Pitch")
  plt.show()

else:
  print('=' * 70)

"""# (CONTINUATION)"""

#@title Standard/Simple Continuation

#@markdown Generation settings

number_of_prime_tokens = 351 # @param {type:"slider", min:3, max:2045, step:3}
number_of_tokens_to_generate = 222 # @param {type:"slider", min:33, max:2045, step:3}
number_of_batches_to_generate = 4 #@param {type:"slider", min:1, max:16, step:1}
temperature = 1 #@param {type:"slider", min:0.1, max:1, step:0.1}

#@markdown Other settings

include_prime_tokens_in_generated_output = True #@param {type:"boolean"}
allow_model_to_stop_generation_if_needed = False #@param {type:"boolean"}
render_MIDI_to_audio = True # @param {type:"boolean"}

print('=' * 70)
print('Los Angeles Music Composer Standard Model Generator')
print('=' * 70)

if allow_model_to_stop_generation_if_needed:
  min_stop_token = 2816
else:
  min_stop_token = 0

outy = melody_chords_f[:number_of_prime_tokens]

inp = [outy] * number_of_batches_to_generate

inp = torch.LongTensor(inp).cuda()

with ctx:
  out = model.module.generate(inp,
                        number_of_tokens_to_generate,
                        temperature=temperature,
                        return_prime=include_prime_tokens_in_generated_output,
                        min_stop_token=min_stop_token,
                        verbose=True)

out0 = out.tolist()
print('=' * 70)
print('Done!')
print('=' * 70)

#======================================================================
print('Rendering results...')

for i in range(number_of_batches_to_generate):

  print('=' * 70)
  print('Batch #', i)
  print('=' * 70)

  out1 = out0[i]

  print('Sample INTs', out1[:12])
  print('=' * 70)

  if len(out) != 0:

      song = out1
      song_f = []
      tim = 0
      dur = 0
      vel = 0
      pitch = 0
      channel = 0

      son = []
      song1 = []

      for s in song:
        if s >= 128 and s < (12*128)+1152:
          son.append(s)
        else:
          if len(son) == 3:
            song1.append(son)
          son = []
          son.append(s)

      for ss in song1:

        tim += ss[0] * 10

        dur = ((ss[1]-128) // 8) * 20
        vel = (((ss[1]-128) % 8)+1) * 15

        channel = (ss[2]-1152) // 128
        pitch = (ss[2]-1152) % 128

        song_f.append(['note', tim, dur, channel, pitch, vel ])

      detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                                output_signature = 'Los Angeles Music Composer',
                                                                output_file_name = '/content/Los-Angeles-Music-Composer-Music-Composition_'+str(i),
                                                                track_name='Project Los Angeles',
                                                                list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 19, 0, 0, 0, 0]
                                                                )
      print('=' * 70)
      print('Displaying resulting composition...')
      print('=' * 70)

      fname = '/content/Los-Angeles-Music-Composer-Music-Composition_'+str(i)

      x = []
      y =[]
      c = []

      colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']

      for s in song_f:
        x.append(s[1] / 1000)
        y.append(s[4])
        c.append(colors[s[3]])

      if render_MIDI_to_audio:
        FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
        display(Audio(str(fname + '.wav'), rate=16000))

      plt.figure(figsize=(14,5))
      ax=plt.axes(title=fname)
      ax.set_facecolor('black')

      plt.scatter(x,y, c=c)
      plt.xlabel("Time")
      plt.ylabel("Pitch")
      plt.show()

"""# (INSTRUMENTS INPAINTING)"""

#@title Pitches/Instruments Inpainting

#@markdown Inpainting settings

#@markdown Select desired instruments to inpaint.

#@markdown Selected instruments MUST BE present in the composition for inpainting to work

#@markdown You can stop the inpainting at any time to render partial results

Piano = False #@param {type:"boolean"}
Guitar = False #@param {type:"boolean"}
Bass = False #@param {type:"boolean"}
Violin = False #@param {type:"boolean"}
Cello = False #@param {type:"boolean"}
Harp = False #@param {type:"boolean"}
Trumpet = False #@param {type:"boolean"}
Clarinet = False #@param {type:"boolean"}
Flute = False #@param {type:"boolean"}
Choir = False #@param {type:"boolean"}
Organ = False #@param {type:"boolean"}

#@markdown Generation settings

number_of_prime_notes = 64 #@param {type:"slider", min:1, max:512, step:1}
number_of_memory_tokens = 4095 # @param {type:"slider", min:6, max:4095, step:3}
number_of_samples_per_inpainted_note = 1 #@param {type:"slider", min:1, max:16, step:1}
temperature = 1 #@param {type:"slider", min:0.1, max:1, step:0.1}

#@markdown Other settings
render_MIDI_to_audio = True # @param {type:"boolean"}


inpaint_instrument = []

if Piano:
  inpaint_instrument.append(0)

if Guitar:
  inpaint_instrument.append(1)

if Bass:
  inpaint_instrument.append(2)

if Violin:
  inpaint_instrument.append(3)

if Cello:
  inpaint_instrument.append(4)

if Harp:
  inpaint_instrument.append(5)

if Trumpet:
  inpaint_instrument.append(6)

if Clarinet:
  inpaint_instrument.append(7)

if Flute:
  inpaint_instrument.append(8)

if Choir:
  inpaint_instrument.append(10)

if Organ:
  inpaint_instrument.append(11)

print('=' * 70)
print('Los Angeles Music Composer Inpainting Model Generator')
print('=' * 70)

out2 = []

for m in melody_chords_f1[:number_of_prime_notes]:
  out2.extend(m)

for i in tqdm.tqdm(range(number_of_prime_notes, len(melody_chords_f1))):

  try:

    if ((melody_chords_f1[i][2]-1152) // 128) in inpaint_instrument:

      out2.extend(melody_chords_f1[i][:2])

      samples = []

      for j in range(number_of_samples_per_inpainted_note):

        inp = torch.LongTensor([out2[-number_of_memory_tokens:]]).cuda()

        with ctx:
          out1 = model.module.generate(inp,
                                1,
                                temperature=temperature,
                                return_prime=True,
                                verbose=False)

          with torch.no_grad():
            with ctx:
              test_loss, test_acc = model(out1)

        samples.append([out1.tolist()[0][-1], test_acc.tolist()])

      accs = [y[1] for y in samples]
      max_acc = max(accs)
      max_acc_sample = samples[accs.index(max_acc)][0]

      out2.extend([max_acc_sample])
    else:
      out2.extend(melody_chords_f1[i])

  except KeyboardInterrupt:
    print('Stopping inpainting...')
    break

  except Exception as e:
    print('Error', e)
    break

print('=' * 70)
print('Done!')
print('=' * 70)

#==================================================

print('Rendering results...')
print('=' * 70)

if len(out2) != 0:

    song = out2
    song_f = []
    time = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0

    son = []
    song1 = []

    for s in song:
      if s >= 128 and s < (12*128)+1152:
        son.append(s)
      else:
        if len(son) == 3:
          song1.append(son)
        son = []
        son.append(s)

    for ss in song1:

      time += ss[0] * 10

      dur = ((ss[1]-128) // 8) * 20
      vel = (((ss[1]-128) % 8)+1) * 15

      channel = (ss[2]-1152) // 128
      pitch = (ss[2]-1152) % 128

      song_f.append(['note', time, dur, channel, pitch, vel ])

    detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                              output_signature = 'Los Angeles Music Composer',
                                                              output_file_name = '/content/Los-Angeles-Music-Composer-Music-Composition',
                                                              track_name='Project Los Angeles',
                                                              list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 19, 0, 0, 0, 0]
                                                              )



    print('=' * 70)
    print('Displaying resulting composition...')
    print('=' * 70)

    fname = '/content/Los-Angeles-Music-Composer-Music-Composition'

    x = []
    y =[]
    c = []

    colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']

    for s in song_f:
      x.append(s[1] / 1000)
      y.append(s[4])
      c.append(colors[s[3]])

    if render_MIDI_to_audio:
      FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
      display(Audio(str(fname + '.wav'), rate=16000))

    plt.figure(figsize=(14,5))
    ax=plt.axes(title=fname)
    ax.set_facecolor('black')

    plt.scatter(x,y, c=c)
    plt.xlabel("Time")
    plt.ylabel("Pitch")
    plt.show()

"""# (HARMONIZATION)"""

#@title Melody Harmonization

#@markdown NOTE: You can stop harmonization at any time to render partial results

melody_instrument = "Violin" #@param ["Piano", "Guitar", "Bass", "Violin", "Cello", "Harp", "Trumpet", "Sax", "Flute", "Choir", "Organ"]
number_of_prime_melody_notes = 4 # @param {type:"slider", min:0, max:50, step:1}
number_of_memory_tokens = 4095 # @param {type:"slider", min:33, max:4095, step:3}
temperature = 1 #@param {type:"slider", min:0.1, max:1, step:0.1}
render_MIDI_to_audio = True # @param {type:"boolean"}

print('=' * 70)
print('Los Angeles Music Composer Melody Harmonization Model Generator')

print('=' * 70)
print('Extracting melody...')
#=======================================================

instruments_list = ["Piano", "Guitar", "Bass", "Violin", "Cello", "Harp", "Trumpet", "Clarinet", "Flute", 'Drums', "Choir", "Organ"]
melody_instrument_number = instruments_list.index(melody_instrument)

melody = []
pe = events_matrix1[0]

for e in events_matrix1:
    if e[3] != 9:

      # Cliping all values...
      time = max(0, min(127, e[1]-pe[1]))
      dur = max(1, min(127, e[2]))
      cha = melody_instrument_number
      ptc = max(1, min(127, e[4]))

      if ptc < 60:
        ptc_aug = (ptc % 12) + 60
      else:
        ptc_aug = ptc

      velocity = round(vel / 15)

      # WRITING EACH NOTE HERE
      dur_vel = (dur * 8) + (velocity-1)
      cha_ptc = (melody_instrument_number * 128) + ptc_aug

      if time != 0:
        melody.append([time, dur_vel+128, cha_ptc+1152])

      pe = e

melody[0][0] = 0

#=======================================================

print('=' * 70)
print('Melody has', len(melody), 'notes')
print('Melody has', len(melody)*3, 'tokens')

print('=' * 70)
print('Starting harmonization...')
print('=' * 70)

outy = []

for m in melody[:number_of_prime_melody_notes]:
  outy.extend(m)

next_chord_time = 0
chord_time_delta = 0
cur_time = 0

for i in tqdm.tqdm(range(number_of_prime_melody_notes, len(melody))):

  try:
    outy.extend(melody[i])

    #====================================================

    outy.extend([0])

    inp = [outy[-number_of_memory_tokens:]]
    inp = torch.LongTensor(inp).cuda()

    with ctx:
      out = model.module.generate(inp,
                            2,
                            temperature=temperature,
                            return_prime=False,
                            verbose=False)

    out0 = out.tolist()[0]



    if (((out0[1]-(1152)) // 128) != melody_instrument_number):
      outy.extend(out0)
    else:
      outy.pop()

    #====================================================

    out0 = [0, 0, 0]
    pout0 = []

    while out0[0] == 0 and out0 != pout0:

      pout0 = out0

      inp = [outy[-number_of_memory_tokens:]]
      inp = torch.LongTensor(inp).cuda()

      with ctx:
        out = model.module.generate(inp,
                              3,
                              temperature=temperature,
                              return_prime=False,
                              verbose=False)

      out0 = out.tolist()[0]

      if out0[0] == 0 and (((out0[2]-(1152)) // 128) != melody_instrument_number) and out0 != pout0:
        outy.extend(out0)

    #====================================================

  except KeyboardInterrupt:
    break

  except Exception as e:
    print('Error', e)
    break

print('=' * 70)
print('Done!')
print('=' * 70)

#======================================================================

print('Rendering results...')
print('=' * 70)

out1 = outy

print('Sample INTs', out1[:12])
print('=' * 70)

if len(out1) != 0:

    song = out1
    song_f = []
    time = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0

    son = []
    song1 = []

    for s in song:
      if s >= 128 and s < (12*128)+1152:
        son.append(s)
      else:
        if len(son) == 3:
          song1.append(son)
        son = []
        son.append(s)

    for ss in song1:

      time += ss[0] * 10

      dur = ((ss[1]-128) // 8) * 20
      vel = (((ss[1]-128) % 8)+1) * 15

      channel = (ss[2]-1152) // 128
      pitch = (ss[2]-1152) % 128

      song_f.append(['note', time, dur, channel, pitch, vel ])

    detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                          output_signature = 'Los Angeles Music Composer',
                                                          output_file_name = '/content/Los-Angeles-Music-Composer-Music-Composition',
                                                          track_name='Project Los Angeles',
                                                          list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 19, 0, 0, 0, 0]
                                                        )


    print('=' * 70)
    print('Displaying resulting composition...')
    print('=' * 70)

    fname = '/content/Los-Angeles-Music-Composer-Music-Composition'

    x = []
    y =[]
    c = []

    colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']

    for s in song_f:
      x.append(s[1] / 1000)
      y.append(s[4])
      c.append(colors[s[3]])

    if render_MIDI_to_audio:
      FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
      display(Audio(str(fname + '.wav'), rate=16000))

    plt.figure(figsize=(14,5))
    ax=plt.axes(title=fname)
    ax.set_facecolor('black')

    plt.scatter(x,y, c=c)
    plt.xlabel("Time")
    plt.ylabel("Pitch")
    plt.show()

"""# Congrats! You did it! :)"""