Volume (#39)

* added volume to the network

* change note input dimension after adding volume

* implement volume in generate

* added jazz to training and generation

* mask replay and volume training

* experiment with downscaling volume based on majority

* midi decode for jazz does not work well hmm

* remove computing merged notes

Author: Taylor Shin

constants.py

@@ -1,7 +1,7 @@
 import os
 
 # Define the musical styles
-styles = ['data/baroque', 'data/classical', 'data/romantic', 'data/modern']
+styles = ['data/baroque', 'data/classical', 'data/romantic', 'data/modern', 'data/jazz']
 # styles = ['data/jazz']
 NUM_STYLES = len(styles)
 
@@ -34,7 +34,7 @@
 OCTAVE_UNITS = 32
 STYLE_UNITS = 32
 BEAT_UNITS = 32
-NOTE_UNITS = 2
+NOTE_UNITS = 3
 TIME_AXIS_UNITS = 300
 NOTE_AXIS_UNITS = 150
 

generate.py

@@ -45,14 +45,16 @@ def build_note_inputs(self, note_features):
         )
 
     def choose(self, prob, n):
-        prob = apply_temperature(prob, self.temperature)
+        vol = prob[n, -1]
+        prob = apply_temperature(prob[n, :-1], self.temperature)
 
         # Flip notes randomly
-        if np.random.random() <= prob[n, 0]:
+        if np.random.random() <= prob[0]:
             self.next_note[n, 0] = 1
-
+            # Apply volume
+            self.next_note[n, 2] = vol
             # Flip articulation
-            if np.random.random() <= prob[n, 1]:
+            if np.random.random() <= prob[1]:
                 self.next_note[n, 1] = 1
 
     def end_time(self, t):
@@ -93,7 +95,7 @@ def process_inputs(ins):
     ins = [np.array(i) for i in ins]
     return ins
 
-def generate(models, num_bars=32, styles=[[1,0,0,0], [0,1,0,0], [0,0,1,0], [0,0,0,1]]):
+def generate(models, num_bars=32, styles=[[1,0,0,0,0], [0,1,0,0,0], [0,0,1,0,0], [0,0,0,1,0], [0,0,0,0,1]]):
     print('Generating with styles:', styles)
 
     _, time_model, note_model = models

midi_util.py

@@ -20,6 +20,7 @@ def midi_encode(note_seq, resolution=NOTES_PER_BEAT, step=1):
 
     play = note_seq[:, :, 0]
     replay = note_seq[:, :, 1]
+    volume = note_seq[:, :, 2]
 
     # The current pattern being played
     current = np.zeros_like(play[0])
@@ -39,7 +40,7 @@ def midi_encode(note_seq, resolution=NOTES_PER_BEAT, step=1):
                     # Was off, but now turned on
                     evt = midi.NoteOnEvent(
                         tick=(tick - last_event_tick) * step,
-                        velocity=int(next_volume * MAX_VELOCITY),
+                        velocity=int(volume[tick][index[0]] * MAX_VELOCITY),
                         pitch=index[0]
                     )
                     track.append(evt)
@@ -62,7 +63,7 @@ def midi_encode(note_seq, resolution=NOTES_PER_BEAT, step=1):
                     track.append(evt_off)
                     evt_on = midi.NoteOnEvent(
                         tick=0,
-                        velocity=int(current[index] * MAX_VELOCITY),
+                        velocity=int(volume[tick][index[0]] * MAX_VELOCITY),
                         pitch=index[0]
                     )
                     track.append(evt_on)
@@ -103,88 +104,88 @@ def midi_decode(pattern,
         step = pattern.resolution // NOTES_PER_BEAT
 
     # Extract all tracks at highest resolution
-    merged_notes = None
     merged_replay = None
+    merged_volume = None
 
     for track in pattern:
         # The downsampled sequences
-        play_sequence = []
         replay_sequence = []
+        volume_sequence = []
 
         # Raw sequences
-        play_buffer = [np.zeros((classes,))]
         replay_buffer = [np.zeros((classes,))]
+        volume_buffer = [np.zeros((classes,))]
 
         for i, event in enumerate(track):
             # Duplicate the last note pattern to wait for next event
             for _ in range(event.tick):
-                play_buffer.append(np.copy(play_buffer[-1]))
                 replay_buffer.append(np.zeros(classes))
+                volume_buffer.append(np.copy(volume_buffer[-1]))
 
                 # Buffer & downscale sequence
-                if len(play_buffer) > step:
-                    # Determine based on majority
-                    notes_sum = np.round(np.sum(play_buffer[:-1], axis=0) / step)
-                    play_sequence.append(play_buffer[0])
-
-                    # Take the max
+                if len(volume_buffer) > step:
+                    # Take the min
                     replay_any = np.minimum(np.sum(replay_buffer[:-1], axis=0), 1)
                     replay_sequence.append(replay_any)
 
+                    # Determine volume on rounded sum
+                    volume_sum = np.round(np.sum(volume_buffer[:-1], axis=0) / step)
+                    volume_sequence.append(volume_sum)
+
                     # Keep the last one (discard things in the middle)
-                    play_buffer = play_buffer[-1:]
                     replay_buffer = replay_buffer[-1:]
+                    volume_buffer = volume_buffer[-1:]
 
             if isinstance(event, midi.EndOfTrackEvent):
                 break
 
             # Modify the last note pattern
             if isinstance(event, midi.NoteOnEvent):
                 pitch, velocity = event.data
-                play_buffer[-1][pitch] = 1 if velocity > 0 else 0
+                volume_buffer[-1][pitch] = velocity / MAX_VELOCITY
 
                 # Check for replay_buffer, which is true if the current note was previously played and needs to be replayed
-                if len(play_buffer) > 1 and play_buffer[-2][pitch] > 0 and play_buffer[-1][pitch] > 0:
+                if len(volume_buffer) > 1 and volume_buffer[-2][pitch] > 0 and volume_buffer[-1][pitch] > 0:
                     replay_buffer[-1][pitch] = 1
                     # Override current volume with previous volume
-                    play_buffer[-1][pitch] = play_buffer[-2][pitch]
+                    volume_buffer[-1][pitch] = volume_buffer[-2][pitch]
 
             if isinstance(event, midi.NoteOffEvent):
                 pitch, velocity = event.data
-                play_buffer[-1][pitch] = 0
+                volume_buffer[-1][pitch] = 0
 
         # Add the remaining
-        play_sequence.append(play_buffer[0])
         replay_any = np.minimum(np.sum(replay_buffer, axis=0), 1)
         replay_sequence.append(replay_any)
+        volume_sequence.append(volume_buffer[0])
 
-        play_sequence = np.array(play_sequence)
         replay_sequence = np.array(replay_sequence)
-        assert len(play_sequence) == len(replay_sequence)
+        volume_sequence = np.array(volume_sequence)
+        assert len(volume_sequence) == len(replay_sequence)
 
-        if merged_notes is None:
-            merged_notes = play_sequence
+        if merged_volume is None:
             merged_replay = replay_sequence
+            merged_volume = volume_sequence
         else:
             # Merge into a single track, padding with zeros of needed
-            if len(play_sequence) > len(merged_notes):
+            if len(volume_sequence) > len(merged_volume):
                 # Swap variables such that merged_notes is always at least
                 # as large as play_sequence
-                tmp = play_sequence
-                play_sequence = merged_notes
-                merged_notes = tmp
-
                 tmp = replay_sequence
                 replay_sequence = merged_replay
                 merged_replay = tmp
 
-            assert len(merged_notes) >= len(play_sequence)
+                tmp = volume_sequence
+                volume_sequence = merged_volume
+                merged_volume = tmp
+
+            assert len(merged_volume) >= len(volume_sequence)
 
-            diff = len(merged_notes) - len(play_sequence)
-            merged_notes += np.pad(play_sequence, ((0, diff), (0, 0)), 'constant')
+            diff = len(merged_volume) - len(volume_sequence)
             merged_replay += np.pad(replay_sequence, ((0, diff), (0, 0)), 'constant')
+            merged_volume += np.pad(volume_sequence, ((0, diff), (0, 0)), 'constant')
 
-    merged = np.stack([merged_notes, merged_replay], axis=2)
+    merged = np.stack([np.ceil(merged_volume), merged_replay, merged_volume], axis=2)
     # Prevent stacking duplicate notes to exceed one.
     merged = np.minimum(merged, 1)
     return merged
@@ -203,13 +204,14 @@ def load_midi(fname):
 
     assert len(note_seq.shape) == 3, note_seq.shape
     assert note_seq.shape[1] == MIDI_MAX_NOTES, note_seq.shape
-    assert note_seq.shape[2] == 2, note_seq.shape
+    assert note_seq.shape[2] == 3, note_seq.shape
     assert (note_seq >= 0).all()
     assert (note_seq <= 1).all()
     return note_seq
 
 if __name__ == '__main__':
     # Test
-    p = midi.read_midifile("out/test_in.mid")
+    # p = midi.read_midifile("out/test_in.mid")
+    p = midi.read_midifile("data/baroque/bach/0864_01.mid")
     p = midi_encode(midi_decode(p))
     midi.write_midifile("out/test_out.mid", p)

model.py

@@ -12,7 +12,12 @@
 from constants import *
 
 def primary_loss(y_true, y_pred):
-    return losses.binary_crossentropy(y_true, y_pred)
+    # 3 separate loss calculations based on if note is played or not
+    played = y_true[:, :, :, 0]
+    bce_note = losses.binary_crossentropy(y_true[:, :, :, 0], y_pred[:, :, :, 0])
+    bce_replay = losses.binary_crossentropy(y_true[:, :, :, 1], tf.multiply(played, y_pred[:, :, :, 1]) + tf.multiply(1 - played, y_true[:, :, :, 1]))
+    mse = losses.mean_squared_error(y_true[:, :, :, 2], tf.multiply(played, y_pred[:, :, :, 2]) + tf.multiply(1 - played, y_true[:, :, :, 2]))
+    return bce_note + bce_replay + mse
 
 def style_loss(y_true, y_pred):
     return 0.5 * losses.categorical_crossentropy(y_true, y_pred)
@@ -89,7 +94,9 @@ def f(notes, beat, style):
 def note_axis(dropout):
     dense_layer_cache = {}
     lstm_layer_cache = {}
-    final_dense = Dense(2, activation='sigmoid', name='note_out')
+    note_dense = Dense(2, activation='sigmoid', name='note_dense')
+    volume_dense = Dense(1, name='volume_dense')
+    # final_dense = Concatenate()([note_dense, volume_dense])
 
     def f(x, chosen, style):
         time_steps = int(x.get_shape()[1])
@@ -120,7 +127,7 @@ def f(x, chosen, style):
             x = Dropout(dropout)(x)
 
         # Primary task
-        return final_dense(x)
+        return Concatenate()([note_dense(x), volume_dense(x)])
     return f
 
 def style_layer(input_dropout):

test.py

@@ -23,7 +23,16 @@ def test_encode(self):
             [0, 0, 0, 0]
         ]
 
-        pattern = midi_encode(np.stack([composition, replay], 2), step=1)
+        volume = [
+            [0, 0.5, 0, 0],
+            [0, 0.5, 0, 0],
+            [0, 0.5, 0, 0.5],
+            [0, 0.5, 0, 0.5],
+            [0, 0, 0, 0.5],
+            [0, 0, 0, 0]
+        ]
+
+        pattern = midi_encode(np.stack([composition, replay, volume], 2), step=1)
         self.assertEqual(pattern.resolution, NOTES_PER_BEAT)
         self.assertEqual(len(pattern), 1)
         track = pattern[0]
@@ -86,7 +95,16 @@ def test_encode_decode(self):
             [0, 0, 0, 0]
         ]
 
-        note_seq = midi_decode(midi_encode(np.stack([composition, replay], 2), step=1), 4, step=1)
+        volume = [
+            [0, 0.5, 0, 0],
+            [0, 0.5, 0, 0],
+            [0, 0.5, 0, 0.5],
+            [0, 0.5, 0, 0.5],
+            [0, 0, 0, 0.5],
+            [0, 0, 0, 0]
+        ]
+
+        note_seq = midi_decode(midi_encode(np.stack([composition, replay, volume], 2), step=1), 4, step=1)
         np.testing.assert_array_equal(composition, note_seq[:, :, 0])
 
     def test_replay_decode(self):
@@ -112,6 +130,31 @@ def test_replay_decode(self):
             [0., 0., 0., 0.]
         ])
 
+
+    def test_volume_decode(self):
+        # Instantiate a MIDI Pattern (contains a list of tracks)
+        pattern = midi.Pattern(resolution=96)
+        # Instantiate a MIDI Track (contains a list of MIDI events)
+        track = midi.Track()
+        # Append the track to the pattern
+        pattern.append(track)
+
+        track.append(midi.NoteOnEvent(tick=0, velocity=24, pitch=0))
+        track.append(midi.NoteOnEvent(tick=96, velocity=89, pitch=1))
+        track.append(midi.NoteOffEvent(tick=0, pitch=0))
+        track.append(midi.NoteOffEvent(tick=48, pitch=1))
+        track.append(midi.EndOfTrackEvent(tick=1))
+
+        note_seq = midi_decode(pattern, 4, step=DEFAULT_RES // 2)
+
+        np.testing.assert_array_almost_equal(note_seq[:, :, 2], [
+            [24/127, 0., 0., 0.],
+            [24/127, 0., 0., 0.],
+            [0., 89/127, 0., 0.],
+            [0., 0., 0., 0.]
+        ], decimal=5)
+
+
     def test_replay_encode_decode(self):
         # TODO: Fix this test
         composition = [
@@ -134,7 +177,17 @@ def test_replay_encode_decode(self):
             [0, 0, 0, 0]
         ]
 
-        note_seq = midi_decode(midi_encode(np.stack([composition, replay], 2), step=2), 4, step=2)
+        volume = [
+            [0, 0.5, 0, 0.5],
+            [0, 0, 0, 0.5],
+            [0, 0, 0, 0.5],
+            [0, 0.5, 0, 0.5],
+            [0, 0.5, 0, 0.5],
+            [0, 0.5, 0, 0.5],
+            [0, 0, 0, 0]
+        ]
+
+        note_seq = midi_decode(midi_encode(np.stack([composition, replay, volume], 2), step=2), 4, step=2)
         np.testing.assert_array_equal(composition, note_seq[:, :, 0])
         # TODO: Downsampling might have caused loss of information
         # np.testing.assert_array_equal(replay, note_seq[:, :, 1])

train.py

@@ -28,7 +28,7 @@ def epoch_cb(epoch, _):
             write_file(os.path.join(SAMPLES_DIR, 'epoch_{}.mid'.format(epoch)), generate(models))
 
     cbs = [
-        ModelCheckpoint(MODEL_FILE, monitor='loss', save_best_only=True),
+        ModelCheckpoint(MODEL_FILE, monitor='loss', save_best_only=True, save_weights_only=True),
         EarlyStopping(monitor='loss', patience=5),
         TensorBoard(log_dir='out/logs', histogram_freq=1)
     ]