musicgen_demo.ipynb

musicgen_demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyN3+Ap7Y81VrJN1XemIO6fj",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/aidiary/30c70fdc2472dd384b63da0622913ec3/musicgen_demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# MusicGen\n",
        "\n",
        "https://github.com/facebookresearch/audiocraft/blob/main/demos/musicgen_demo.ipynb"
      ],
      "metadata": {
        "id": "1UgiNHsKa6JJ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yo7B_SrHa1tQ"
      },
      "outputs": [],
      "source": [
        "%pip install audiocraft"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 訓練済みモデルをロード"
      ],
      "metadata": {
        "id": "kFRS7RPbbgKN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from audiocraft.models import MusicGen\n",
        "\n",
        "model = MusicGen.get_pretrained(\"facebook/musicgen-small\")"
      ],
      "metadata": {
        "id": "hhBeW6YGbe6K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.set_generation_params(\n",
        "    use_sampling=True,\n",
        "    top_k=250,\n",
        "    duration=30\n",
        ")"
      ],
      "metadata": {
        "id": "1RH7j1b2b5aG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Music Continuation"
      ],
      "metadata": {
        "id": "PmfJAFDics1-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import math\n",
        "import torchaudio\n",
        "import torch\n",
        "from audiocraft.utils.notebook import display_audio"
      ],
      "metadata": {
        "id": "FoTO5iodcqtb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_bip_bip(bip_duration=0.125, frequency=440, duration=0.5, sample_rate=32000, device=\"cuda\"):\n",
        "    t = torch.arange(int(duration * sample_rate), device=\"cuda\", dtype=torch.float) / sample_rate\n",
        "    wav = torch.cos(2 * math.pi * 440 * t)[None]\n",
        "    tp = (t % (2 * bip_duration)) / (2 * bip_duration)\n",
        "    envelope = (tp >= 0.5).float()\n",
        "    return wav * envelope"
      ],
      "metadata": {
        "id": "3JEkKeGYcz_d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# ２回のビープに続く音楽を合成する\n",
        "res = model.generate_continuation(\n",
        "    get_bip_bip(0.125).expand(2, -1, -1),\n",
        "    32000,\n",
        "    [\"Jazz jazz and only jazz\", \"Heartful EDM with beautiful synths and chords\"],\n",
        "    progress=True\n",
        ")\n",
        "display_audio(res, 32000)"
      ],
      "metadata": {
        "id": "0gRedhd0dMQO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "display_audio(res, 32000)"
      ],
      "metadata": {
        "id": "pQfx8XwSdf5x"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# バッハ「トッカータとフーガニ短調」の最初の2秒から続く音楽を合成\n",
        "prompt_waveform, prompt_sr = torchaudio.load(\"https://github.com/facebookresearch/audiocraft/raw/main/assets/bach.mp3\")"
      ],
      "metadata": {
        "id": "7EuZ-Z-4eR59"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prompt_sr"
      ],
      "metadata": {
        "id": "trabJFYMervK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "display_audio(prompt_waveform, prompt_sr)"
      ],
      "metadata": {
        "id": "QHlUp6bSfM8u"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prompt_duration = 2\n",
        "prompt_waveform = prompt_waveform[..., :int(prompt_duration* prompt_sr)]\n",
        "display_audio(prompt_waveform, prompt_sr)"
      ],
      "metadata": {
        "id": "jpZbdV_jfORZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output = model.generate_continuation(prompt_waveform,\n",
        "                                     prompt_sr,\n",
        "                                     progress=True,\n",
        "                                     return_tokens=True)"
      ],
      "metadata": {
        "id": "6aQg9JktfiW4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "display_audio(output[0], sample_rate=32000)"
      ],
      "metadata": {
        "id": "jfFUm7Xsf8Y8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Text-conditional Generation"
      ],
      "metadata": {
        "id": "P7sWwDDqg501"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from audiocraft.utils.notebook import display_audio"
      ],
      "metadata": {
        "id": "r25_g_imgIOE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output = model.generate(\n",
        "    descriptions=[\n",
        "        \"drum and bass beat with intense percussions\"\n",
        "    ],\n",
        "    progress=True,\n",
        "    return_tokens=True\n",
        ")\n",
        "display_audio(output[0], sample_rate=32000)"
      ],
      "metadata": {
        "id": "_khKNv1dhAZq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Melody-conditional Generation"
      ],
      "metadata": {
        "id": "doW5_7Ikhj-7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torchaudio\n",
        "from audiocraft.utils.notebook import display_audio"
      ],
      "metadata": {
        "id": "D7hWuKGKhNmS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = MusicGen.get_pretrained('facebook/musicgen-melody')\n",
        "model.set_generation_params(duration=8)"
      ],
      "metadata": {
        "id": "U50Ey68fhm1f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "melody_waveform, sr = torchaudio.load(\"https://github.com/facebookresearch/audiocraft/raw/main/assets/bach.mp3\")\n",
        "melody_waveform = melody_waveform.unsqueeze(0).repeat(2, 1, 1)\n",
        "melody_waveform.shape"
      ],
      "metadata": {
        "id": "XgtGsBdkhos1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output = model.generate_with_chroma(\n",
        "    descriptions=[\n",
        "        '80s pop track with bassy drums and synth',\n",
        "        '90s rock song with loud guitars and heavy drums',\n",
        "    ],\n",
        "    melody_wavs=melody_waveform,\n",
        "    melody_sample_rate=sr,\n",
        "    progress=True, return_tokens=True\n",
        ")\n",
        "display_audio(output[0], sample_rate=32000)"
      ],
      "metadata": {
        "id": "s5SvSUmkhzos"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "aeScsmBCh-cm"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}