inference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "### 1. Import libraries"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "4308f645195af7fa"
  },
  {
   "cell_type": "code",
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"4\"\n",
    "from audioldm2.p2p_pipeline import AudioLDM2Pipeline\n",
    "from diffusers import DDIMScheduler, DDIMInverseScheduler\n",
    "import torch\n",
    "import nltk\n",
    "import IPython\n",
    "import soundfile as sf\n",
    "import datetime\n",
    "from lpmc.music_captioning import captioning\n",
    "from audioldm2.embedding_calculator import EmbeddingCalculator\n",
    "\n",
    "model_ckpt = \"cvssp/audioldm2\"\n",
    "pipeline = AudioLDM2Pipeline.from_pretrained(model_ckpt,\n",
    "                                             torch_dtype=torch.float32).to(\"cuda\")\n",
    "pipeline.scheduler = DDIMScheduler.from_config(pipeline.scheduler.config)\n",
    "pipeline.inverse_scheduler = DDIMInverseScheduler.from_config(pipeline.scheduler.config)\n",
    "RANDOM_SEED = 42"
   ],
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "is_executing": true
    }
   },
   "id": "efee19619352611c",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### 2. Real audio inversion (optional)",
   "id": "dcd5f3c0b7695531"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "before_concept = \"piano\"\n",
    "after_concept = \"saxophone\""
   ],
   "id": "5dd07370cf1f0895"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "# get text input\n",
    "audio_path = \"sample_audio/bach_anh114.wav\"\n",
    "caption = captioning.captioning(audio_path)\n",
    "prompt = EmbeddingCalculator.postprocessing_caption(caption, before_concept)\n",
    "print(prompt)"
   ],
   "id": "6b2c203fb35d2799"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### 3. Calculating Δ",
   "id": "ee863b4d70ef8b15"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "embedding_calculator = EmbeddingCalculator(embedding_model=pipeline, prompt_length=len(nltk.word_tokenize(prompt)))\n",
    "\n",
    "source_embeddings, generated_source_embeddings, target_embeddings, generated_target_embeddings = embedding_calculator(\n",
    "    before_concept,\n",
    "    after_concept,)"
   ],
   "id": "3dab5b671e48673a"
  },
  {
   "cell_type": "code",
   "source": [
    "# get latent\n",
    "g_cpu = torch.Generator().manual_seed(RANDOM_SEED)\n",
    "inv_latents, audio_origin = pipeline.invert(prompt, audio_path=audio_path, generator=g_cpu, guidance_scale=1)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "f43031881ebd23cb",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 4. Diffusion"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "c58cf726b97f1c98"
  },
  {
   "cell_type": "code",
   "source": [
    "g_cpu = torch.Generator().manual_seed(RANDOM_SEED)\n",
    "audios_before_, audios_after_ = pipeline(\n",
    "    prompt,\n",
    "    negative_prompt=\"low quality\",\n",
    "    source_embeddings=source_embeddings,\n",
    "    generated_source_embeddings=generated_source_embeddings,\n",
    "    target_embeddings=target_embeddings,\n",
    "    generated_target_embeddings=generated_target_embeddings,\n",
    "    num_inference_steps=100,\n",
    "    audio_length_in_s=10,\n",
    "    guidance_scale=3.5,\n",
    "    cross_attention_guidance_amount=0.04,\n",
    "    generator=g_cpu,\n",
    "    # num_waveforms_per_prompt=3,\n",
    "    before_concept=before_concept,\n",
    "    after_concept=after_concept,\n",
    "    latents=inv_latents,\n",
    "    )"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "2284019f8292f5a7",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 5. Display results"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "c09389aeef55e52a"
  },
  {
   "cell_type": "code",
   "source": [
    "audios_before, audios_after = audios_before_.audios[0], audios_after_.audios[0]\n",
    "\n",
    "IPython.display.display(IPython.display.Audio(audios_before, rate=16000))\n",
    "IPython.display.display(IPython.display.Audio(audios_after, rate=16000))\n",
    "# save\n",
    "\n",
    "\n",
    "current_time = datetime.datetime.now().strftime(\"%m%d%H%M%S\")\n",
    "\n",
    "sf.write(f\"outputs/{current_time}_{before_concept}_to_{after_concept}_before.wav\", audios_before, 16000)\n",
    "sf.write(f\"outputs/{current_time}_{before_concept}_to_{after_concept}_after.wav\", audios_after, 16000)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "initial_id",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}