Los_Angeles_MIDI_Dataset_Search_and_Explore.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "gradient": {
          "editing": false,
          "id": "ac5a4cf0-d9d2-47b5-9633-b53f8d99a4d2",
          "kernelId": ""
        },
        "id": "SiTIpPjArIyr"
      },
      "source": [
        "# Los Angeles MIDI Dataset: Search and Explore (ver. 4.0)\n",
        "\n",
        "***\n",
        "\n",
        "Powered by tegridy-tools: https://github.com/asigalov61/tegridy-tools\n",
        "\n",
        "***\n",
        "\n",
        "#### Project Los Angeles\n",
        "\n",
        "#### Tegridy Code 2023\n",
        "\n",
        "***"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "gradient": {
          "editing": false,
          "id": "fa0a611c-1803-42ae-bdf6-a49b5a4e781b",
          "kernelId": ""
        },
        "id": "gOd93yV0sGd2"
      },
      "source": [
        "# (SETUP ENVIRONMENT)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "gradient": {
          "editing": false,
          "id": "a1a45a91-d909-4fd4-b67a-5e16b971d179",
          "kernelId": ""
        },
        "id": "fX12Yquyuihc",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Install all dependencies (run only once per session)\n",
        "!git clone --depth 1 https://github.com/asigalov61/Los-Angeles-MIDI-Dataset\n",
        "!pip install huggingface_hub\n",
        "!pip install matplotlib\n",
        "!pip install sklearn\n",
        "!pip install tqdm\n",
        "!apt install fluidsynth #Pip does not work for some reason. Only apt works\n",
        "!pip install midi2audio"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "gradient": {
          "editing": false,
          "id": "b8207b76-9514-4c07-95db-95a4742e52c5",
          "kernelId": ""
        },
        "id": "z7n9vnKmug1J",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Import all needed modules\n",
        "\n",
        "print('Loading core modules...')\n",
        "import os\n",
        "import copy\n",
        "from collections import Counter\n",
        "import random\n",
        "import pickle\n",
        "from tqdm import tqdm\n",
        "import pprint\n",
        "import statistics\n",
        "\n",
        "from joblib import Parallel, delayed\n",
        "import multiprocessing\n",
        "\n",
        "if not os.path.exists('/content/LAMD'):\n",
        "    os.makedirs('/content/LAMD')\n",
        "\n",
        "print('Loading MIDI.py module...')\n",
        "os.chdir('/content/Los-Angeles-MIDI-Dataset')\n",
        "import MIDI\n",
        "\n",
        "print('Loading aux modules...')\n",
        "from sklearn.metrics import pairwise_distances, pairwise\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from midi2audio import FluidSynth\n",
        "from IPython.display import Audio, display\n",
        "\n",
        "from huggingface_hub import hf_hub_download\n",
        "\n",
        "from google.colab import files\n",
        "\n",
        "os.chdir('/content/')\n",
        "print('Done!')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (PREP DATA)"
      ],
      "metadata": {
        "id": "Erj-5wMNA1Jd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Unzip LAMDa data\n",
        "%cd /content/Los-Angeles-MIDI-Dataset/META-DATA\n",
        "\n",
        "print('=' * 70)\n",
        "print('Unzipping META-DATA...Please wait...')\n",
        "\n",
        "!cat LAMDa_META_DATA.zip* > LAMDa_META_DATA.zip\n",
        "print('=' * 70)\n",
        "\n",
        "!unzip -j LAMDa_META_DATA.zip\n",
        "print('=' * 70)\n",
        "\n",
        "print('Done! Enjoy! :)')\n",
        "print('=' * 70)\n",
        "%cd /content/\n",
        "\n",
        "#================================================\n",
        "\n",
        "%cd /content/Los-Angeles-MIDI-Dataset/TOTALS\n",
        "\n",
        "print('=' * 70)\n",
        "print('Unzipping TOTALS...Please wait...')\n",
        "\n",
        "!unzip -j LAMDa_TOTALS.zip\n",
        "print('=' * 70)\n",
        "\n",
        "print('Done! Enjoy! :)')\n",
        "print('=' * 70)\n",
        "%cd /content/"
      ],
      "metadata": {
        "cellView": "form",
        "id": "Shs-6PR6ie7-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Load LAMDa data\n",
        "print('=' * 70)\n",
        "print('Loading LAMDa data...Please wait...')\n",
        "print('=' * 70)\n",
        "print('Loading LAMDa META-DATA...')\n",
        "meta_data = pickle.load(open('/content/Los-Angeles-MIDI-Dataset/META-DATA/LAMDa_META_DATA.pickle', 'rb'))\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "print('Loading LAMDa TOTALS...')\n",
        "totals = pickle.load(open('/content/Los-Angeles-MIDI-Dataset/TOTALS/LAMDa_TOTALS.pickle', 'rb'))\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "print('Enjoy!')\n",
        "print('=' * 70)"
      ],
      "metadata": {
        "id": "ML1CecXtjiGE",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (PREP MIDI DATASET)"
      ],
      "metadata": {
        "id": "GDKQys8pFgtj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Download the dataset\n",
        "print('=' * 70)\n",
        "print('Downloading Los Angeles MIDI Dataset...Please wait...')\n",
        "print('=' * 70)\n",
        "\n",
        "hf_hub_download(repo_id='projectlosangeles/Los-Angeles-MIDI-Dataset',\n",
        "                filename='Los-Angeles-MIDI-Dataset-Ver-4-0-CC-BY-NC-SA.zip',\n",
        "                repo_type=\"dataset\",\n",
        "                local_dir='/content/LAMD',\n",
        "                local_dir_use_symlinks=False)\n",
        "print('=' * 70)\n",
        "print('Done! Enjoy! :)')\n",
        "print('=' * 70)"
      ],
      "metadata": {
        "id": "U_KPkpWMCkNO",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Unzip the dataset\n",
        "%cd /content/LAMD\n",
        "\n",
        "print('=' * 70)\n",
        "print('Unzipping Los Angeles MIDI Dataset...Please wait...')\n",
        "!unzip 'Los-Angeles-MIDI-Dataset-Ver-4-0-CC-BY-NC-SA.zip'\n",
        "print('=' * 70)\n",
        "\n",
        "print('Done! Enjoy! :)')\n",
        "print('=' * 70)\n",
        "%cd /content/"
      ],
      "metadata": {
        "cellView": "form",
        "id": "mKFlrD98FMAP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Create dataset files list\n",
        "print('=' * 70)\n",
        "print('Creating dataset files list...')\n",
        "dataset_addr = \"/content/LAMD/MIDIs\"\n",
        "\n",
        "# os.chdir(dataset_addr)\n",
        "filez = list()\n",
        "for (dirpath, dirnames, filenames) in os.walk(dataset_addr):\n",
        "    filez += [os.path.join(dirpath, file) for file in filenames]\n",
        "\n",
        "if filez == []:\n",
        "    print('Could not find any MIDI files. Please check Dataset dir...')\n",
        "    print('=' * 70)\n",
        "\n",
        "print('=' * 70)\n",
        "print('Randomizing file list...')\n",
        "random.shuffle(filez)\n",
        "print('=' * 70)\n",
        "\n",
        "LAMD_files_list = []\n",
        "\n",
        "for f in tqdm(filez):\n",
        "  LAMD_files_list.append([f.split('/')[-1].split('.mid')[0], f])\n",
        "print('Done!')\n",
        "print('=' * 70)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "nMRMnn7rHBg5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (PLOT TOTALS)"
      ],
      "metadata": {
        "id": "ta6K3AOOqhpM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Plot totals from MIDI matrixes (legacy)\n",
        "\n",
        "cos_sim = pairwise.cosine_similarity(\n",
        "      totals[0][0][4]\n",
        "  )\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.imshow(cos_sim, cmap=\"inferno\", interpolation=\"none\")\n",
        "im_ratio = 1\n",
        "plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)\n",
        "plt.title('Times')\n",
        "plt.xlabel(\"Position\")\n",
        "plt.ylabel(\"Position\")\n",
        "plt.tight_layout()\n",
        "plt.plot()\n",
        "\n",
        "cos_sim = pairwise.cosine_similarity(\n",
        "      totals[0][0][5]\n",
        "  )\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.imshow(cos_sim, cmap=\"inferno\", interpolation=\"none\")\n",
        "im_ratio = 1\n",
        "plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)\n",
        "plt.title('Durations')\n",
        "plt.xlabel(\"Position\")\n",
        "plt.ylabel(\"Position\")\n",
        "plt.tight_layout()\n",
        "plt.plot()\n",
        "\n",
        "cos_sim = pairwise.cosine_similarity(\n",
        "      totals[0][0][6]\n",
        "  )\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.imshow(cos_sim, cmap=\"inferno\", interpolation=\"none\")\n",
        "im_ratio = 1\n",
        "plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)\n",
        "plt.title('Channels')\n",
        "plt.xlabel(\"Position\")\n",
        "plt.ylabel(\"Position\")\n",
        "plt.tight_layout()\n",
        "plt.plot()\n",
        "\n",
        "cos_sim = pairwise.cosine_similarity(\n",
        "      totals[0][0][7]\n",
        "  )\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.imshow(cos_sim, cmap=\"inferno\", interpolation=\"none\")\n",
        "im_ratio = 1\n",
        "plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)\n",
        "plt.title('Instruments')\n",
        "plt.xlabel(\"Position\")\n",
        "plt.ylabel(\"Position\")\n",
        "plt.tight_layout()\n",
        "plt.plot()\n",
        "\n",
        "cos_sim = pairwise.cosine_similarity(\n",
        "      totals[0][0][8]\n",
        "  )\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.imshow(cos_sim, cmap=\"inferno\", interpolation=\"none\")\n",
        "im_ratio = 1\n",
        "plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)\n",
        "plt.title('Pitches')\n",
        "plt.xlabel(\"Position\")\n",
        "plt.ylabel(\"Position\")\n",
        "plt.tight_layout()\n",
        "plt.plot()\n",
        "\n",
        "cos_sim = pairwise.cosine_similarity(\n",
        "      totals[0][0][9]\n",
        "  )\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.imshow(cos_sim, cmap=\"inferno\", interpolation=\"none\")\n",
        "im_ratio = 1\n",
        "plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)\n",
        "plt.title('Velocities')\n",
        "plt.xlabel(\"Position\")\n",
        "plt.ylabel(\"Position\")\n",
        "plt.tight_layout()\n",
        "plt.plot()"
      ],
      "metadata": {
        "cellView": "form",
        "id": "Tk-3Q3UnqkpL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Plot totals from MIDI metadata\n",
        "\n",
        "#===============================================================================\n",
        "\n",
        "pitches_counts_totals = [0] * 128\n",
        "\n",
        "for m in tqdm(meta_data):\n",
        "  for mm in m[1][10][1]:\n",
        "    if mm[0] < 128:\n",
        "      pitches_counts_totals[mm[0]] += mm[1]\n",
        "\n",
        "y = range(128)\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.plot(y, pitches_counts_totals)\n",
        "\n",
        "plt.title('MIDI Instruments Pitches')\n",
        "plt.xlabel(\"Pitch\")\n",
        "plt.ylabel(\"Count\")\n",
        "plt.tight_layout()\n",
        "plt.plot()\n",
        "\n",
        "sim_mat = [ [0]*128 for i in range(128)]\n",
        "x = 0\n",
        "\n",
        "for p in pitches_counts_totals:\n",
        "  y = 0\n",
        "  for pp in pitches_counts_totals:\n",
        "\n",
        "    sim_mat[x][y] = min(10, (p / pp))\n",
        "    y += 1\n",
        "\n",
        "  x += 1\n",
        "\n",
        "cos_sim = pairwise.cosine_similarity(\n",
        "      sim_mat\n",
        "  )\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.imshow(sim_mat, cmap=\"inferno\", interpolation=\"none\")\n",
        "im_ratio = 1\n",
        "plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)\n",
        "plt.title('MIDI Drums Pitches')\n",
        "plt.xlabel(\"Pitch\")\n",
        "plt.ylabel(\"Count\")\n",
        "plt.tight_layout()\n",
        "plt.plot()\n",
        "\n",
        "#===============================================================================\n",
        "\n",
        "pitches_counts_totals = [1] * 128\n",
        "\n",
        "\n",
        "for m in tqdm(meta_data):\n",
        "  for mm in m[1][10][1]:\n",
        "    if mm[0] > 128:\n",
        "      pitches_counts_totals[mm[0] % 128] += mm[1]\n",
        "\n",
        "y = range(128)\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.plot(y, pitches_counts_totals)\n",
        "\n",
        "plt.title('MIDI Drums Pitches')\n",
        "plt.xlabel(\"Pitch\")\n",
        "plt.ylabel(\"Count\")\n",
        "plt.tight_layout()\n",
        "plt.plot()\n",
        "\n",
        "sim_mat = [ [0]*128 for i in range(128)]\n",
        "x = 0\n",
        "\n",
        "for p in pitches_counts_totals:\n",
        "  y = 0\n",
        "  for pp in pitches_counts_totals:\n",
        "\n",
        "    sim_mat[x][y] = min(10, (p / pp))\n",
        "    y += 1\n",
        "\n",
        "  x += 1\n",
        "\n",
        "cos_sim = pairwise.cosine_similarity(\n",
        "      sim_mat\n",
        "  )\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.imshow(sim_mat, cmap=\"inferno\", interpolation=\"none\")\n",
        "im_ratio = 1\n",
        "plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)\n",
        "plt.title('MIDI Drums Pitches')\n",
        "plt.xlabel(\"Pitch\")\n",
        "plt.ylabel(\"Count\")\n",
        "plt.tight_layout()\n",
        "plt.plot()\n",
        "\n",
        "#===============================================================================\n",
        "\n",
        "patches_counts_totals = [0] * 256\n",
        "\n",
        "\n",
        "for m in tqdm(meta_data):\n",
        "  for mm in m[1][12][1]:\n",
        "    patches_counts_totals[mm[0]] += mm[1]\n",
        "\n",
        "\n",
        "y = range(128)\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.plot(y, patches_counts_totals[:128])\n",
        "\n",
        "plt.title('MIDI Patches')\n",
        "plt.xlabel(\"Patch\")\n",
        "plt.ylabel('Count')\n",
        "plt.tight_layout()\n",
        "plt.plot()"
      ],
      "metadata": {
        "cellView": "form",
        "id": "NLo2pUSQjpgg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (LOAD SOURCE MIDI)"
      ],
      "metadata": {
        "id": "jiVsfC0u7aAQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Load source MIDI\n",
        "\n",
        "full_path_to_source_MIDI = \"/content/Los-Angeles-MIDI-Dataset/Come-To-My-Window-Modified-Sample-MIDI.mid\" #@param {type:\"string\"}\n",
        "render_MIDI_to_audio = False #@param {type:\"boolean\"}\n",
        "\n",
        "#=================================================================================\n",
        "\n",
        "f = full_path_to_source_MIDI\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading MIDI file...')\n",
        "\n",
        "#==================================================\n",
        "\n",
        "score = MIDI.midi2score(open(f, 'rb').read())\n",
        "\n",
        "events_matrix = []\n",
        "\n",
        "track_count = 0\n",
        "\n",
        "for s in score:\n",
        "\n",
        "    if track_count > 0:\n",
        "        track = s\n",
        "        track.sort(key=lambda x: x[1])\n",
        "        events_matrix.extend(track)\n",
        "    else:\n",
        "        midi_ticks = s\n",
        "\n",
        "    track_count += 1\n",
        "\n",
        "events_matrix.sort(key=lambda x: x[1])\n",
        "\n",
        "mult_pitches_counts = []\n",
        "\n",
        "for i in range(-6, 6):\n",
        "\n",
        "  events_matrix1 = []\n",
        "\n",
        "  for e in events_matrix:\n",
        "\n",
        "    ev = copy.deepcopy(e)\n",
        "\n",
        "    if e[0] == 'note':\n",
        "        if e[3] == 9:\n",
        "            ev[4] = ((e[4] % 128) + 128)\n",
        "        else:\n",
        "          ev[4] = ((e[4] % 128) + i)\n",
        "\n",
        "        events_matrix1.append(ev)\n",
        "\n",
        "  pitches_counts = [[y[0],y[1]] for y in Counter([y[4] for y in events_matrix1 if y[0] == 'note']).most_common()]\n",
        "  pitches_counts.sort(key=lambda x: x[0], reverse=True)\n",
        "\n",
        "  mult_pitches_counts.append(pitches_counts)\n",
        "\n",
        "patches_list = sorted(list(set([y[3] for y in events_matrix if y[0] == 'patch_change'])))\n",
        "\n",
        "\n",
        "#==================================================\n",
        "\n",
        "ms_score = MIDI.midi2ms_score(open(f, 'rb').read())\n",
        "\n",
        "ms_events_matrix = []\n",
        "\n",
        "itrack1 = 1\n",
        "\n",
        "while itrack1 < len(ms_score):\n",
        "    for event in ms_score[itrack1]:\n",
        "        if event[0] == 'note':\n",
        "            ms_events_matrix.append(event)\n",
        "    itrack1 += 1\n",
        "\n",
        "ms_events_matrix.sort(key=lambda x: x[1])\n",
        "\n",
        "\n",
        "chords = []\n",
        "pe = ms_events_matrix[0]\n",
        "cho = []\n",
        "for e in ms_events_matrix:\n",
        "    if (e[1] - pe[1]) == 0:\n",
        "      if e[3] != 9:\n",
        "        if (e[4] % 12) not in cho:\n",
        "          cho.append(e[4] % 12)\n",
        "    else:\n",
        "      if len(cho) > 0:\n",
        "        chords.append(sorted(cho))\n",
        "      cho = []\n",
        "      if e[3] != 9:\n",
        "        if (e[4] % 12) not in cho:\n",
        "          cho.append(e[4] % 12)\n",
        "\n",
        "    pe = e\n",
        "\n",
        "if len(cho) > 0:\n",
        "    chords.append(sorted(cho))\n",
        "\n",
        "ms_chords_counts = sorted([[list(key), val] for key,val in Counter([tuple(c) for c in chords if len(c) > 1]).most_common()], reverse=True, key = lambda x: x[1])\n",
        "\n",
        "times = []\n",
        "pt = ms_events_matrix[0][1]\n",
        "start = True\n",
        "for e in ms_events_matrix:\n",
        "    if (e[1]-pt) != 0 or start == True:\n",
        "        times.append((e[1]-pt))\n",
        "        start = False\n",
        "    pt = e[1]\n",
        "\n",
        "durs = [e[2] for e in ms_events_matrix]\n",
        "vels = [e[5] for e in ms_events_matrix]\n",
        "\n",
        "avg_time = int(sum(times) / len(times))\n",
        "avg_dur = int(sum(durs) / len(durs))\n",
        "\n",
        "mode_time = statistics.mode(times)\n",
        "mode_dur = statistics.mode(durs)\n",
        "\n",
        "median_time = int(statistics.median(times))\n",
        "median_dur = int(statistics.median(durs))\n",
        "\n",
        "#==================================================\n",
        "\n",
        "print('=' * 70)\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "\n",
        "#============================================\n",
        "# MIDI rendering code\n",
        "#============================================\n",
        "\n",
        "print('Rendering source MIDI...')\n",
        "print('=' * 70)\n",
        "\n",
        "ms_score = MIDI.midi2ms_score(open(f, 'rb').read())\n",
        "\n",
        "itrack = 1\n",
        "song_f = []\n",
        "\n",
        "while itrack < len(ms_score):\n",
        "    for event in ms_score[itrack]:\n",
        "        if event[0] == 'note':\n",
        "            song_f.append(event)\n",
        "    itrack += 1\n",
        "\n",
        "song_f.sort(key=lambda x: x[1])\n",
        "\n",
        "fname = f.split('.mid')[0]\n",
        "\n",
        "x = []\n",
        "y =[]\n",
        "c = []\n",
        "\n",
        "colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver', 'aqua', 'azure', 'bisque', 'coral']\n",
        "\n",
        "for s in song_f:\n",
        "  x.append(s[1] / 1000)\n",
        "  y.append(s[4])\n",
        "  c.append(colors[s[3]])\n",
        "\n",
        "if render_MIDI_to_audio:\n",
        "  FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "  display(Audio(str(fname + '.wav'), rate=16000))\n",
        "\n",
        "plt.figure(figsize=(14,5))\n",
        "ax=plt.axes(title=fname)\n",
        "ax.set_facecolor('black')\n",
        "\n",
        "plt.scatter(x,y, c=c)\n",
        "plt.xlabel(\"Time\")\n",
        "plt.ylabel(\"Pitch\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "z7mm6WsDmuRi",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (SEARCH AND EXPLORE)"
      ],
      "metadata": {
        "id": "LRnuphuiqc5F"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title MIDI Pitches Search\n",
        "\n",
        "#@markdown NOTE: You can stop the search at any time to render partial results\n",
        "\n",
        "#@markdown Match ratio control option\n",
        "\n",
        "maximum_match_ratio_to_search_for = 1 #@param {type:\"slider\", min:0, max:1, step:0.01}\n",
        "\n",
        "#@markdown MIDI pitches search options\n",
        "\n",
        "pitches_counts_cutoff_threshold_ratio = 0 #@param {type:\"slider\", min:0, max:1, step:0.05}\n",
        "search_transposed_pitches = False #@param {type:\"boolean\"}\n",
        "skip_exact_matches = False #@param {type:\"boolean\"}\n",
        "\n",
        "#@markdown Additional search options\n",
        "\n",
        "add_pitches_counts_ratios = False #@param {type:\"boolean\"}\n",
        "add_timings_ratios = False #@param {type:\"boolean\"}\n",
        "add_durations_ratios = False #@param {type:\"boolean\"}\n",
        "\n",
        "#@markdown Other options\n",
        "\n",
        "render_MIDI_to_audio = False #@param {type:\"boolean\"}\n",
        "download_MIDI = False #@param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('MIDI Pitches Search')\n",
        "print('=' * 70)\n",
        "\n",
        "final_ratios = []\n",
        "\n",
        "for d in tqdm(meta_data):\n",
        "\n",
        "  try:\n",
        "    p_counts = d[1][10][1]\n",
        "    p_counts.sort(reverse = True, key = lambda x: x[1])\n",
        "    max_p_count = p_counts[0][1]\n",
        "    trimmed_p_counts = [y for y in p_counts if y[1] >= (max_p_count * pitches_counts_cutoff_threshold_ratio)]\n",
        "    total_p_counts = sum([y[1] for y in trimmed_p_counts])\n",
        "\n",
        "    if search_transposed_pitches:\n",
        "      search_pitches = mult_pitches_counts\n",
        "    else:\n",
        "      search_pitches = [mult_pitches_counts[6]]\n",
        "\n",
        "    #===================================================\n",
        "\n",
        "    ratios_list = []\n",
        "\n",
        "    #===================================================\n",
        "\n",
        "    atrat = [0]\n",
        "\n",
        "    if add_timings_ratios:\n",
        "\n",
        "      source_times = [avg_time,\n",
        "                      median_time,\n",
        "                      mode_time]\n",
        "\n",
        "      match_times = meta_data[0][1][3][1]\n",
        "\n",
        "      times_ratios = []\n",
        "\n",
        "      for i in range(len(source_times)):\n",
        "        maxtratio = max(source_times[i], match_times[i])\n",
        "        mintratio = min(source_times[i], match_times[i])\n",
        "        times_ratios.append(mintratio / maxtratio)\n",
        "\n",
        "      avg_times_ratio = sum(times_ratios) / len(times_ratios)\n",
        "\n",
        "      atrat[0] = avg_times_ratio\n",
        "\n",
        "    #===================================================\n",
        "\n",
        "    adrat = [0]\n",
        "\n",
        "    if add_durations_ratios:\n",
        "\n",
        "      source_durs = [avg_dur,\n",
        "                      median_dur,\n",
        "                      mode_dur]\n",
        "\n",
        "      match_durs = meta_data[0][1][4][1]\n",
        "\n",
        "      durs_ratios = []\n",
        "\n",
        "      for i in range(len(source_durs)):\n",
        "        maxtratio = max(source_durs[i], match_durs[i])\n",
        "        mintratio = min(source_durs[i], match_durs[i])\n",
        "        durs_ratios.append(mintratio / maxtratio)\n",
        "\n",
        "      avg_durs_ratio = sum(durs_ratios) / len(durs_ratios)\n",
        "\n",
        "      adrat[0] = avg_durs_ratio\n",
        "\n",
        "    #===================================================\n",
        "\n",
        "    for m in search_pitches:\n",
        "\n",
        "      sprat = []\n",
        "\n",
        "      m.sort(reverse = True, key = lambda x: x[1])\n",
        "      max_pitches_count = m[0][1]\n",
        "      trimmed_pitches_counts = [y for y in m if y[1] >= (max_pitches_count * pitches_counts_cutoff_threshold_ratio)]\n",
        "      total_pitches_counts = sum([y[1] for y in trimmed_pitches_counts])\n",
        "\n",
        "      same_pitches = set([T[0] for T in trimmed_p_counts]) & set([m[0] for m in trimmed_pitches_counts])\n",
        "      num_same_pitches = len(same_pitches)\n",
        "\n",
        "      if num_same_pitches == len(trimmed_pitches_counts):\n",
        "        same_pitches_ratio = (num_same_pitches / len(trimmed_p_counts))\n",
        "      else:\n",
        "        same_pitches_ratio = (num_same_pitches / max(len(trimmed_p_counts), len(trimmed_pitches_counts)))\n",
        "\n",
        "      if skip_exact_matches:\n",
        "        if same_pitches_ratio == 1:\n",
        "          same_pitches_ratio = 0\n",
        "\n",
        "      sprat.append(same_pitches_ratio)\n",
        "\n",
        "      #===================================================\n",
        "\n",
        "      spcrat = [0]\n",
        "\n",
        "      if add_pitches_counts_ratios:\n",
        "\n",
        "        same_trimmed_p_counts = sorted([T for T in trimmed_p_counts if T[0] in same_pitches], reverse = True)\n",
        "        same_trimmed_pitches_counts = sorted([T for T in trimmed_pitches_counts if T[0] in same_pitches], reverse = True)\n",
        "\n",
        "        same_trimmed_p_counts_ratios = [[s[0], s[1] / total_p_counts] for s in same_trimmed_p_counts]\n",
        "        same_trimmed_pitches_counts_ratios = [[s[0], s[1] / total_pitches_counts] for s in same_trimmed_pitches_counts]\n",
        "\n",
        "        same_pitches_counts_ratios = []\n",
        "\n",
        "        for i in range(len(same_trimmed_p_counts_ratios)):\n",
        "          mincratio = min(same_trimmed_p_counts_ratios[i][1], same_trimmed_pitches_counts_ratios[i][1])\n",
        "          maxcratio = max(same_trimmed_p_counts_ratios[i][1], same_trimmed_pitches_counts_ratios[i][1])\n",
        "          same_pitches_counts_ratios.append([same_trimmed_p_counts_ratios[i][0], mincratio / maxcratio])\n",
        "\n",
        "        same_counts_ratios = [s[1] for s in same_pitches_counts_ratios]\n",
        "\n",
        "        if len(same_counts_ratios) > 0:\n",
        "          avg_same_pitches_counts_ratio = sum(same_counts_ratios) / len(same_counts_ratios)\n",
        "        else:\n",
        "          avg_same_pitches_counts_ratio = 0\n",
        "\n",
        "        spcrat[0] = avg_same_pitches_counts_ratio\n",
        "\n",
        "      #===================================================\n",
        "\n",
        "      r_list = [sprat[0]]\n",
        "\n",
        "      if add_pitches_counts_ratios:\n",
        "        r_list.append(spcrat[0])\n",
        "\n",
        "      if add_timings_ratios:\n",
        "        r_list.append(atrat[0])\n",
        "\n",
        "      if add_durations_ratios:\n",
        "        r_list.append(adrat[0])\n",
        "\n",
        "      ratios_list.append(r_list)\n",
        "\n",
        "    #===================================================\n",
        "\n",
        "    avg_ratios_list = []\n",
        "\n",
        "    for r in ratios_list:\n",
        "      avg_ratios_list.append(sum(r) / len(r))\n",
        "\n",
        "    #===================================================\n",
        "\n",
        "    final_ratio = max(avg_ratios_list)\n",
        "\n",
        "    if final_ratio > maximum_match_ratio_to_search_for:\n",
        "        final_ratio = 0\n",
        "\n",
        "    final_ratios.append(final_ratio)\n",
        "\n",
        "    #===================================================\n",
        "\n",
        "  except KeyboardInterrupt:\n",
        "    break\n",
        "\n",
        "  except Exception as e:\n",
        "    print('WARNING !!!')\n",
        "    print('=' * 70)\n",
        "    print('Error detected:', e)\n",
        "    final_ratios.append(0)\n",
        "    print('=' * 70)\n",
        "    break\n",
        "\n",
        "max_ratio = max(final_ratios)\n",
        "max_ratio_index = final_ratios.index(max_ratio)\n",
        "\n",
        "print('FOUND')\n",
        "print('=' * 70)\n",
        "print('Match ratio', max_ratio)\n",
        "print('MIDI file name', meta_data[max_ratio_index][0])\n",
        "print('=' * 70)\n",
        "pprint.pprint(['Sample metadata entries', meta_data[max_ratio_index][1][:8]], compact = True)\n",
        "print('=' * 70)\n",
        "\n",
        "#============================================\n",
        "# MIDI rendering code\n",
        "#============================================\n",
        "\n",
        "print('Rendering source MIDI...')\n",
        "print('=' * 70)\n",
        "\n",
        "fn = meta_data[max_ratio_index][0]\n",
        "fn_idx = [y[0] for y in LAMD_files_list].index(fn)\n",
        "\n",
        "f = LAMD_files_list[fn_idx][1]\n",
        "\n",
        "ms_score = MIDI.midi2ms_score(open(f, 'rb').read())\n",
        "\n",
        "itrack = 1\n",
        "song_f = []\n",
        "\n",
        "while itrack < len(ms_score):\n",
        "    for event in ms_score[itrack]:\n",
        "        if event[0] == 'note':\n",
        "            song_f.append(event)\n",
        "    itrack += 1\n",
        "\n",
        "song_f.sort(key=lambda x: x[1])\n",
        "\n",
        "fname = f.split('.mid')[0]\n",
        "\n",
        "x = []\n",
        "y =[]\n",
        "c = []\n",
        "\n",
        "colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver', 'aqua', 'azure', 'bisque', 'coral']\n",
        "\n",
        "for s in song_f:\n",
        "  x.append(s[1] / 1000)\n",
        "  y.append(s[4])\n",
        "  c.append(colors[s[3]])\n",
        "\n",
        "if render_MIDI_to_audio:\n",
        "  FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "  display(Audio(str(fname + '.wav'), rate=16000))\n",
        "\n",
        "plt.figure(figsize=(14,5))\n",
        "ax=plt.axes(title=fname)\n",
        "ax.set_facecolor('black')\n",
        "\n",
        "plt.scatter(x,y, c=c)\n",
        "plt.xlabel(\"Time\")\n",
        "plt.ylabel(\"Pitch\")\n",
        "plt.show()\n",
        "\n",
        "#==============================================\n",
        "\n",
        "if download_MIDI:\n",
        "  print('=' * 70)\n",
        "  print('Downloading MIDI file', str(fn) + '.mid')\n",
        "  files.download(f)\n",
        "  print('=' * 70)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "C-P71_9ojh8G"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title MIDI Chords Search\n",
        "\n",
        "#@markdown NOTE: You can stop the search at any time to render partial results\n",
        "\n",
        "maximum_match_ratio_to_search_for = 1 #@param {type:\"slider\", min:0, max:1, step:0.01}\n",
        "chords_counts_cutoff_threshold_ratio = 0 #@param {type:\"slider\", min:0, max:1, step:0.05}\n",
        "skip_exact_matches = False #@param {type:\"boolean\"}\n",
        "render_MIDI_to_audio = False #@param {type:\"boolean\"}\n",
        "download_MIDI = False #@param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('MIDI Chords Search')\n",
        "print('=' * 70)\n",
        "\n",
        "ratios = []\n",
        "\n",
        "for d in tqdm(meta_data):\n",
        "\n",
        "  try:\n",
        "\n",
        "    c_counts = d[1][8][1]\n",
        "    if len(c_counts) == 0:\n",
        "      c_counts = copy.deepcopy([[[0, 0], 0]])\n",
        "\n",
        "    c_counts.sort(reverse = True, key = lambda x: x[0][1])\n",
        "    max_c_count = c_counts[0][1]\n",
        "    trimmed_c_counts = [y for y in c_counts if y[1] >= (max_c_count * chords_counts_cutoff_threshold_ratio)]\n",
        "    trimmed_c_counts.sort(reverse = True, key = lambda x: x[1])\n",
        "\n",
        "    max_chords_count = ms_chords_counts[0][1]\n",
        "    trimmed_chords_counts = [y for y in ms_chords_counts if y[1] >= (max_chords_count * chords_counts_cutoff_threshold_ratio)]\n",
        "\n",
        "    num_same_chords = len(set([tuple(T[0]) for T in trimmed_c_counts]) & set([tuple(t[0]) for t in trimmed_chords_counts]))\n",
        "\n",
        "    if num_same_chords == len(trimmed_chords_counts):\n",
        "      same_chords_ratio = (num_same_chords / len(trimmed_c_counts))\n",
        "    else:\n",
        "      same_chords_ratio = (num_same_chords / max(len(trimmed_c_counts), len(trimmed_chords_counts)))\n",
        "\n",
        "    if skip_exact_matches:\n",
        "      if same_chords_ratio == 1:\n",
        "        same_chords_ratio = 0\n",
        "\n",
        "    if same_chords_ratio > maximum_match_ratio_to_search_for:\n",
        "      same_chords_ratio = 0\n",
        "\n",
        "    ratios.append(same_chords_ratio)\n",
        "\n",
        "  except KeyboardInterrupt:\n",
        "    break\n",
        "\n",
        "  except Exception as e:\n",
        "    print('WARNING !!!')\n",
        "    print('=' * 70)\n",
        "    print('Error detected:', e)\n",
        "    ratios.append(0)\n",
        "    print('=' * 70)\n",
        "    continue\n",
        "\n",
        "max_ratio = max(ratios)\n",
        "max_ratio_index = ratios.index(max(ratios))\n",
        "\n",
        "print('FOUND')\n",
        "print('=' * 70)\n",
        "print('Match ratio', max_ratio)\n",
        "print('MIDI file name', meta_data[max_ratio_index][0])\n",
        "print('=' * 70)\n",
        "pprint.pprint(['Sample metadata entries', meta_data[max_ratio_index][1][:8]], compact = True)\n",
        "print('=' * 70)\n",
        "\n",
        "#============================================\n",
        "# MIDI rendering code\n",
        "#============================================\n",
        "\n",
        "print('Rendering source MIDI...')\n",
        "print('=' * 70)\n",
        "\n",
        "fn = meta_data[max_ratio_index][0]\n",
        "fn_idx = [y[0] for y in LAMD_files_list].index(fn)\n",
        "\n",
        "f = LAMD_files_list[fn_idx][1]\n",
        "\n",
        "ms_score = MIDI.midi2ms_score(open(f, 'rb').read())\n",
        "\n",
        "itrack = 1\n",
        "song_f = []\n",
        "\n",
        "while itrack < len(ms_score):\n",
        "    for event in ms_score[itrack]:\n",
        "        if event[0] == 'note':\n",
        "            song_f.append(event)\n",
        "    itrack += 1\n",
        "\n",
        "song_f.sort(key=lambda x: x[1])\n",
        "\n",
        "fname = f.split('.mid')[0]\n",
        "\n",
        "x = []\n",
        "y =[]\n",
        "c = []\n",
        "\n",
        "colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver', 'aqua', 'azure', 'bisque', 'coral']\n",
        "\n",
        "for s in song_f:\n",
        "  x.append(s[1] / 1000)\n",
        "  y.append(s[4])\n",
        "  c.append(colors[s[3]])\n",
        "\n",
        "if render_MIDI_to_audio:\n",
        "  FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "  display(Audio(str(fname + '.wav'), rate=16000))\n",
        "\n",
        "plt.figure(figsize=(14,5))\n",
        "ax=plt.axes(title=fname)\n",
        "ax.set_facecolor('black')\n",
        "\n",
        "plt.scatter(x,y, c=c)\n",
        "plt.xlabel(\"Time\")\n",
        "plt.ylabel(\"Pitch\")\n",
        "plt.show()\n",
        "\n",
        "#==============================================\n",
        "\n",
        "if download_MIDI:\n",
        "  print('=' * 70)\n",
        "  print('Downloading MIDI file', str(fn) + '.mid')\n",
        "  files.download(f)\n",
        "  print('=' * 70)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "HBV0CtPAA8nF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title MIDI Patches Search\n",
        "\n",
        "#@markdown NOTE: You can stop the search at any time to render partial results\n",
        "\n",
        "maximum_match_ratio_to_search_for = 1 #@param {type:\"slider\", min:0, max:1, step:0.01}\n",
        "skip_exact_matches = False #@param {type:\"boolean\"}\n",
        "render_MIDI_to_audio = False #@param {type:\"boolean\"}\n",
        "download_MIDI = False #@param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('MIDI Patches Search')\n",
        "print('=' * 70)\n",
        "\n",
        "ratios = []\n",
        "\n",
        "for d in tqdm(meta_data):\n",
        "\n",
        "  try:\n",
        "\n",
        "    p_list= d[1][11][1]\n",
        "\n",
        "    num_same_patches = len(set(p_list) & set(patches_list))\n",
        "\n",
        "    if len(set(p_list + patches_list)) > 0:\n",
        "\n",
        "      if num_same_patches == len(patches_list):\n",
        "        same_patches_ratio = num_same_patches / len(p_list)\n",
        "      else:\n",
        "        same_patches_ratio = num_same_patches / max(len(p_list), len(patches_list))\n",
        "\n",
        "    else:\n",
        "      same_patches_ratio = 0\n",
        "\n",
        "    if skip_exact_matches:\n",
        "      if same_patches_ratio == 1:\n",
        "        same_patches_ratio = 0\n",
        "\n",
        "    if same_patches_ratio > maximum_match_ratio_to_search_for:\n",
        "      same_patches_ratio = 0\n",
        "\n",
        "    ratios.append(same_patches_ratio)\n",
        "\n",
        "  except KeyboardInterrupt:\n",
        "    break\n",
        "\n",
        "  except Exception as e:\n",
        "    print('WARNING !!!')\n",
        "    print('=' * 70)\n",
        "    print('Error detected:', e)\n",
        "    ratios.append(0)\n",
        "    print('=' * 70)\n",
        "    continue\n",
        "\n",
        "max_ratio = max(ratios)\n",
        "max_ratio_index = ratios.index(max(ratios))\n",
        "\n",
        "print('FOUND')\n",
        "print('=' * 70)\n",
        "print('Match ratio', max_ratio)\n",
        "print('MIDI file name', meta_data[max_ratio_index][0])\n",
        "print('=' * 70)\n",
        "print('Found MIDI patches list', meta_data[max_ratio_index][1][12][1])\n",
        "print('=' * 70)\n",
        "\n",
        "#============================================\n",
        "# MIDI rendering code\n",
        "#============================================\n",
        "\n",
        "print('Rendering source MIDI...')\n",
        "print('=' * 70)\n",
        "\n",
        "fn = meta_data[max_ratio_index][0]\n",
        "fn_idx = [y[0] for y in LAMD_files_list].index(fn)\n",
        "\n",
        "f = LAMD_files_list[fn_idx][1]\n",
        "\n",
        "ms_score = MIDI.midi2ms_score(open(f, 'rb').read())\n",
        "\n",
        "itrack = 1\n",
        "song_f = []\n",
        "\n",
        "while itrack < len(ms_score):\n",
        "    for event in ms_score[itrack]:\n",
        "        if event[0] == 'note':\n",
        "            song_f.append(event)\n",
        "    itrack += 1\n",
        "\n",
        "song_f.sort(key=lambda x: x[1])\n",
        "\n",
        "fname = f.split('.mid')[0]\n",
        "\n",
        "x = []\n",
        "y =[]\n",
        "c = []\n",
        "\n",
        "colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver', 'aqua', 'azure', 'bisque', 'coral']\n",
        "\n",
        "for s in song_f:\n",
        "  x.append(s[1] / 1000)\n",
        "  y.append(s[4])\n",
        "  c.append(colors[s[3]])\n",
        "\n",
        "if render_MIDI_to_audio:\n",
        "  FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "  display(Audio(str(fname + '.wav'), rate=16000))\n",
        "\n",
        "plt.figure(figsize=(14,5))\n",
        "ax=plt.axes(title=fname)\n",
        "ax.set_facecolor('black')\n",
        "\n",
        "plt.scatter(x,y, c=c)\n",
        "plt.xlabel(\"Time\")\n",
        "plt.ylabel(\"Pitch\")\n",
        "plt.show()\n",
        "\n",
        "#==============================================\n",
        "\n",
        "if download_MIDI:\n",
        "  print('=' * 70)\n",
        "  print('Downloading MIDI file', str(fn) + '.mid')\n",
        "  files.download(f)\n",
        "  print('=' * 70)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "JmV8VNj-tz1m"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Metadata Search\n",
        "\n",
        "#@markdown You can search the metadata by search query or by MIDI md5 hash file name\n",
        "\n",
        "search_query = \"Come To My Window\" #@param {type:\"string\"}\n",
        "md5_hash_MIDI_file_name = \"d9a7e1c6a375b8e560155a5977fc10f8\" #@param {type:\"string\"}\n",
        "case_sensitive_search = False #@param {type:\"boolean\"}\n",
        "\n",
        "fields_to_search = ['track_name',\n",
        "                    'text_event',\n",
        "                    'lyric',\n",
        "                    'copyright_text_event',\n",
        "                    'marker',\n",
        "                    'text_event_08',\n",
        "                    'text_event_09',\n",
        "                    'text_event_0a',\n",
        "                    'text_event_0b',\n",
        "                    'text_event_0c',\n",
        "                    'text_event_0d',\n",
        "                    'text_event_0e',\n",
        "                    'text_event_0f',\n",
        "                    ]\n",
        "\n",
        "print('=' * 70)\n",
        "print('Los Angeles MIDI Dataset Metadata Search')\n",
        "print('=' * 70)\n",
        "print('Searching...')\n",
        "print('=' * 70)\n",
        "\n",
        "if md5_hash_MIDI_file_name != '':\n",
        "  for d in tqdm(meta_data):\n",
        "    try:\n",
        "      if d[0] == md5_hash_MIDI_file_name:\n",
        "        print('Found!')\n",
        "        print('=' * 70)\n",
        "        print('Metadata index:', meta_data.index(d))\n",
        "        print('MIDI file name:', meta_data[meta_data.index(d)][0])\n",
        "        print('-' * 70)\n",
        "        pprint.pprint(['Result:', d[1][:16]], compact = True)\n",
        "        print('=' * 70)\n",
        "        break\n",
        "\n",
        "    except KeyboardInterrupt:\n",
        "      print('Ending search...')\n",
        "      print('=' * 70)\n",
        "      break\n",
        "\n",
        "    except Exception as e:\n",
        "      print('WARNING !!!')\n",
        "      print('=' * 70)\n",
        "      print('Error detected:', e)\n",
        "      print('=' * 70)\n",
        "      continue\n",
        "\n",
        "  if d[0] != md5_hash_MIDI_file_name:\n",
        "    print('Not found!')\n",
        "    print('=' * 70)\n",
        "    print('md5 hash was not found!')\n",
        "    print('Ending search...')\n",
        "    print('=' * 70)\n",
        "\n",
        "else:\n",
        "  for d in tqdm(meta_data):\n",
        "    try:\n",
        "      for dd in d[1]:\n",
        "        if dd[0] in fields_to_search:\n",
        "          if case_sensitive_search:\n",
        "            if str(search_query) in str(dd[2]):\n",
        "              print('Found!')\n",
        "              print('=' * 70)\n",
        "              print('Metadata index:', meta_data.index(d))\n",
        "              print('MIDI file name:', meta_data[meta_data.index(d)][0])\n",
        "              print('-' * 70)\n",
        "              pprint.pprint(['Result:', dd[2][:16]], compact = True)\n",
        "              print('=' * 70)\n",
        "\n",
        "          else:\n",
        "            if str(search_query).lower() in str(dd[2]).lower():\n",
        "              print('Found!')\n",
        "              print('=' * 70)\n",
        "              print('Metadata index:', meta_data.index(d))\n",
        "              print('MIDI file name:', meta_data[meta_data.index(d)][0])\n",
        "              print('-' * 70)\n",
        "              pprint.pprint(['Result:', dd[2][:16]], compact = True)\n",
        "              print('=' * 70)\n",
        "\n",
        "    except KeyboardInterrupt:\n",
        "      print('Ending search...')\n",
        "      print('=' * 70)\n",
        "      break\n",
        "\n",
        "    except:\n",
        "      print('Ending search...')\n",
        "      print('=' * 70)\n",
        "      break"
      ],
      "metadata": {
        "cellView": "form",
        "id": "98JuBEdMqIE3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (MIDI FILE PLAYER)"
      ],
      "metadata": {
        "id": "ih7bAzjv2amL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title MIDI file player\n",
        "\n",
        "#@markdown NOTE: You can use md5 hash file name or full MIDI file path to play it\n",
        "\n",
        "md5_hash_MIDI_file_name = \"d9a7e1c6a375b8e560155a5977fc10f8\" #@param {type:\"string\"}\n",
        "full_path_to_MIDI = \"/content/Los-Angeles-MIDI-Dataset/Come-To-My-Window-Modified-Sample-MIDI.mid\" #@param {type:\"string\"}\n",
        "render_MIDI_to_audio = False #@param {type:\"boolean\"}\n",
        "download_MIDI = False #@param {type:\"boolean\"}\n",
        "\n",
        "#============================================\n",
        "# MIDI rendering code\n",
        "#============================================\n",
        "\n",
        "print('=' * 70)\n",
        "print('MIDI file player')\n",
        "print('=' * 70)\n",
        "\n",
        "try:\n",
        "\n",
        "  if os.path.exists(full_path_to_MIDI):\n",
        "    f = full_path_to_MIDI\n",
        "    print('Using full path to MIDI')\n",
        "\n",
        "  else:\n",
        "    fn = md5_hash_MIDI_file_name\n",
        "    fn_idx = [y[0] for y in LAMD_files_list].index(fn)\n",
        "    f = LAMD_files_list[fn_idx][1]\n",
        "\n",
        "    print('Using md5 hash filename')\n",
        "\n",
        "  print('=' * 70)\n",
        "  print('Rendering MIDI...')\n",
        "  print('=' * 70)\n",
        "\n",
        "  ms_score = MIDI.midi2ms_score(open(f, 'rb').read())\n",
        "\n",
        "  itrack = 1\n",
        "  song_f = []\n",
        "\n",
        "  while itrack < len(ms_score):\n",
        "      for event in ms_score[itrack]:\n",
        "          if event[0] == 'note':\n",
        "              song_f.append(event)\n",
        "      itrack += 1\n",
        "\n",
        "  song_f.sort(key=lambda x: x[1])\n",
        "\n",
        "  fname = f.split('.mid')[0]\n",
        "\n",
        "  x = []\n",
        "  y =[]\n",
        "  c = []\n",
        "\n",
        "  colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver', 'aqua', 'azure', 'bisque', 'coral']\n",
        "\n",
        "  for s in song_f:\n",
        "    x.append(s[1] / 1000)\n",
        "    y.append(s[4])\n",
        "    c.append(colors[s[3]])\n",
        "\n",
        "  if render_MIDI_to_audio:\n",
        "    FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "    display(Audio(str(fname + '.wav'), rate=16000))\n",
        "\n",
        "  plt.figure(figsize=(14,5))\n",
        "  ax=plt.axes(title=fname)\n",
        "  ax.set_facecolor('black')\n",
        "\n",
        "  plt.scatter(x,y, c=c)\n",
        "  plt.xlabel(\"Time\")\n",
        "  plt.ylabel(\"Pitch\")\n",
        "  plt.show()\n",
        "\n",
        "  #==============================================\n",
        "\n",
        "  if download_MIDI:\n",
        "    print('=' * 70)\n",
        "    print('Downloading MIDI file', str(fn) + '.mid')\n",
        "    files.download(f)\n",
        "    print('=' * 70)\n",
        "\n",
        "except:\n",
        "  print('File not found!!!')\n",
        "  print('Check the filename!')\n",
        "  print('=' * 70)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "rfnp7dQw2egM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (COLAB MIDI FILES LOCATOR/DOWNLOADER)"
      ],
      "metadata": {
        "id": "pk1Wf3SfARmy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Loacate and/or download desired MIDI files by MIDI md5 hash file names\n",
        "\n",
        "MIDI_md5_hash_file_name_1 = \"d9a7e1c6a375b8e560155a5977fc10f8\" #@param {type:\"string\"}\n",
        "MIDI_md5_hash_file_name_2 = \"\" #@param {type:\"string\"}\n",
        "MIDI_md5_hash_file_name_3 = \"\" #@param {type:\"string\"}\n",
        "MIDI_md5_hash_file_name_4 = \"\" #@param {type:\"string\"}\n",
        "MIDI_md5_hash_file_name_5 = \"\" #@param {type:\"string\"}\n",
        "download_located_files = False #@param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('MIDI files locator and downloader')\n",
        "print('=' * 70)\n",
        "\n",
        "md5_list = []\n",
        "\n",
        "if MIDI_md5_hash_file_name_1 != '':\n",
        "  md5_list.append(MIDI_md5_hash_file_name_1)\n",
        "\n",
        "if MIDI_md5_hash_file_name_2 != '':\n",
        "  md5_list.append(MIDI_md5_hash_file_name_2)\n",
        "\n",
        "if MIDI_md5_hash_file_name_3 != '':\n",
        "  md5_list.append(MIDI_md5_hash_file_name_3)\n",
        "\n",
        "if MIDI_md5_hash_file_name_4 != '':\n",
        "  md5_list.append(MIDI_md5_hash_file_name_4)\n",
        "\n",
        "if MIDI_md5_hash_file_name_5 != '':\n",
        "  md5_list.append(MIDI_md5_hash_file_name_5)\n",
        "\n",
        "if len(md5_list) > 0:\n",
        "  for m in md5_list:\n",
        "    try:\n",
        "\n",
        "      fn = m\n",
        "      fn_idx = [y[0] for y in LAMD_files_list].index(fn)\n",
        "      f = LAMD_files_list[fn_idx][1]\n",
        "\n",
        "      print('Found md5 hash file name', m)\n",
        "\n",
        "      location_str = ''\n",
        "\n",
        "      fl = f.split('/')\n",
        "      for fa in fl[:-1]:\n",
        "        if fa != '' and fa != 'content':\n",
        "          location_str += '/'\n",
        "          location_str += str(fa)\n",
        "\n",
        "      print('Colab location/folder', location_str)\n",
        "\n",
        "      if download_located_files:\n",
        "        print('Downloading MIDI file', str(m) + '.mid')\n",
        "        files.download(f)\n",
        "\n",
        "      print('=' * 70)\n",
        "\n",
        "    except:\n",
        "      print('md5 hash file name', m, 'not found!!!')\n",
        "      print('Check the file name!')\n",
        "      print('=' * 70)\n",
        "      continue\n",
        "\n",
        "else:\n",
        "  print('No md5 hash file names were specified!')\n",
        "  print('Check input!')\n",
        "  print('=' * 70)\n"
      ],
      "metadata": {
        "cellView": "form",
        "id": "MCehs2qxAaOl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (CUSTOM ANALYSIS TEMPLATE)"
      ],
      "metadata": {
        "id": "jNH-J2ry7sLj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Los Angeles MIDI Dataset Reader\n",
        "\n",
        "print('=' * 70)\n",
        "print('Los Angeles MIDI Dataset Reader')\n",
        "print('=' * 70)\n",
        "print('Starting up...')\n",
        "print('=' * 70)\n",
        "\n",
        "###########\n",
        "\n",
        "print('Loading MIDI files...')\n",
        "print('This may take a while on a large dataset in particular.')\n",
        "\n",
        "dataset_addr = \"/content/LAMD/MIDIs\"\n",
        "\n",
        "# os.chdir(dataset_addr)\n",
        "filez = list()\n",
        "for (dirpath, dirnames, filenames) in os.walk(dataset_addr):\n",
        "    filez += [os.path.join(dirpath, file) for file in filenames]\n",
        "\n",
        "if filez == []:\n",
        "    print('Could not find any MIDI files. Please check Dataset dir...')\n",
        "    print('=' * 70)\n",
        "\n",
        "print('=' * 70)\n",
        "print('Randomizing file list...')\n",
        "random.shuffle(filez)\n",
        "print('=' * 70)\n",
        "\n",
        "###########\n",
        "\n",
        "START_FILE_NUMBER = 0\n",
        "LAST_SAVED_BATCH_COUNT = 0\n",
        "\n",
        "input_files_count = START_FILE_NUMBER\n",
        "files_count = LAST_SAVED_BATCH_COUNT\n",
        "\n",
        "stats = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "\n",
        "print('Reading MIDI files. Please wait...')\n",
        "print('=' * 70)\n",
        "\n",
        "for f in tqdm(filez[START_FILE_NUMBER:]):\n",
        "    try:\n",
        "        input_files_count += 1\n",
        "\n",
        "        fn = os.path.basename(f)\n",
        "        fn1 = fn.split('.mid')[0]\n",
        "\n",
        "        #=======================================================\n",
        "        # START PROCESSING\n",
        "        #=======================================================\n",
        "\n",
        "        # Convering MIDI to score with MIDI.py module\n",
        "        score = MIDI.midi2score(open(f, 'rb').read())\n",
        "\n",
        "        events_matrix = []\n",
        "\n",
        "        itrack = 1\n",
        "\n",
        "        while itrack < len(score):\n",
        "            for event in score[itrack]:\n",
        "              events_matrix.append(event)\n",
        "            itrack += 1\n",
        "\n",
        "        # Sorting...\n",
        "        events_matrix.sort(key=lambda x: x[1])\n",
        "\n",
        "        if len(events_matrix) > 0:\n",
        "\n",
        "          #=======================================================\n",
        "          # INSERT YOUR CUSTOM ANAYLSIS CODE RIGHT HERE\n",
        "          #=======================================================\n",
        "\n",
        "          # Processed files counter\n",
        "          files_count += 1\n",
        "\n",
        "          # Saving every 5000 processed files\n",
        "          if files_count % 10000 == 0:\n",
        "            print('=' * 70)\n",
        "            print('Processed so far:', files_count, 'out of', input_files_count, '===', files_count / input_files_count, 'good files ratio')\n",
        "            print('=' * 70)\n",
        "\n",
        "    except KeyboardInterrupt:\n",
        "        print('Saving current progress and quitting...')\n",
        "        break\n",
        "\n",
        "    except Exception as ex:\n",
        "        print('WARNING !!!')\n",
        "        print('=' * 70)\n",
        "        print('Bad MIDI:', f)\n",
        "        print('Error detected:', ex)\n",
        "        print('=' * 70)\n",
        "        continue\n",
        "\n",
        "print('=' * 70)\n",
        "print('Final files counts:', files_count, 'out of', input_files_count, '===', files_count / input_files_count, 'good files ratio')\n",
        "print('=' * 70)\n",
        "\n",
        "print('Resulting Stats:')\n",
        "print('=' * 70)\n",
        "print('Total good processed MIDI files:', files_count)\n",
        "print('=' * 70)\n",
        "print('Done!')\n",
        "print('=' * 70)"
      ],
      "metadata": {
        "id": "xrmnVUkXGsUi",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YzCMd94Tu_gz"
      },
      "source": [
        "# Congrats! You did it! :)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    },
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}