use new models for now

notebooks/audio_diffusion_pipeline.ipynb

@@ -46,7 +46,7 @@
     "from datasets import load_dataset\n",
     "from IPython.display import Audio\n",
     "from librosa.beat import beat_track\n",
-    "from diffusers import DiffusionPipeline, Mel"
+    "from diffusers import DiffusionPipeline"
    ]
   },
   {
@@ -56,8 +56,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "mel = Mel()\n",
-    "sample_rate = mel.get_sample_rate()\n",
     "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
     "generator = torch.Generator(device=device)"
    ]
@@ -91,7 +89,7 @@
     "\n",
     "#@markdown teticio/audio-diffusion-instrumental-hiphop-256 - trained on instrumental hiphop\n",
     "\n",
-    "model_id = \"teticio/audio-diffusion-256\"  #@param [\"teticio/audio-diffusion-256\", \"teticio/audio-diffusion-breaks-256\", \"audio-diffusion-instrumenal-hiphop-256\", \"teticio/audio-diffusion-ddim-256\"]"
+    "model_id = \"teticio/audio-diffusion-256-new\"  #@param [\"teticio/audio-diffusion-256\", \"teticio/audio-diffusion-breaks-256\", \"audio-diffusion-instrumenal-hiphop-256\", \"teticio/audio-diffusion-ddim-256\"]"
    ]
   },
   {
@@ -101,7 +99,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "audio_diffusion = DiffusionPipeline.from_pretrained(model_id).to(device)"
+    "audio_diffusion = DiffusionPipeline.from_pretrained(model_id).to(device)\n",
+    "mel = audio_diffusion.mel\n",
+    "sample_rate = mel.get_sample_rate()"
    ]
   },
   {
@@ -150,7 +150,7 @@
     "    seed = generator.seed()\n",
     "    print(f'Seed = {seed}')\n",
     "    generator.manual_seed(seed)\n",
-    "    output = audio_diffusion(mel=mel, generator=generator)\n",
+    "    output = audio_diffusion(generator=generator)\n",
     "    image = output.images[0]\n",
     "    audio = output.audios[0, 0]\n",
     "    display(image)\n",
@@ -187,7 +187,7 @@
    "source": [
     "seed = 2391504374279719  #@param {type:\"integer\"}\n",
     "generator.manual_seed(seed)\n",
-    "output = audio_diffusion(mel=mel, generator=generator)\n",
+    "output = audio_diffusion(generator=generator)\n",
     "image = output.images[0]\n",
     "audio = output.audios[0, 0]\n",
     "display(image)\n",
@@ -206,7 +206,7 @@
     "start_step = 500  #@param {type:\"slider\", min:0, max:1000, step:10}\n",
     "track = loop_it(audio, sample_rate, loops=1)\n",
     "for variation in range(12):\n",
-    "    output = audio_diffusion(mel=mel, raw_audio=audio, start_step=start_step)\n",
+    "    output = audio_diffusion(raw_audio=audio, start_step=start_step)\n",
     "    image2 = output.images[0]\n",
     "    audio2 = output.audios[0, 0]\n",
     "    display(image2)\n",
@@ -235,8 +235,7 @@
     "overlap_samples = overlap_secs * sample_rate\n",
     "track = audio\n",
     "for variation in range(12):\n",
-    "    output = audio_diffusion(mel=mel,\n",
-    "                             raw_audio=audio[-overlap_samples:],\n",
+    "    output = audio_diffusion(raw_audio=audio[-overlap_samples:],\n",
     "                             start_step=start_step,\n",
     "                             mask_start_secs=overlap_secs)\n",
     "    image2 = output.images[0]\n",
@@ -306,8 +305,7 @@
     "        # Normalize and re-insert generated audio\n",
     "        audio[:overlap_samples] = audio2[-overlap_samples:] * np.max(\n",
     "            audio[:overlap_samples]) / np.max(audio2[-overlap_samples:])\n",
-    "    output = audio_diffusion(mel=mel,\n",
-    "                             raw_audio=audio,\n",
+    "    output = audio_diffusion(raw_audio=audio,\n",
     "                             start_step=start_step,\n",
     "                             generator=generator,\n",
     "                             mask_start_secs=overlap_secs * not_first)\n",
@@ -334,8 +332,7 @@
    "source": [
     "sample = 3  #@param {type:\"integer\"}\n",
     "raw_audio = track_audio[sample * stride:sample * stride + slice_size]\n",
-    "output = audio_diffusion(mel=mel,\n",
-    "                         raw_audio=raw_audio,\n",
+    "output = audio_diffusion(raw_audio=raw_audio,\n",
     "                         mask_start_secs=1,\n",
     "                         mask_end_secs=1,\n",
     "                         step_generator=torch.Generator(device=device))\n",
@@ -359,7 +356,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "audio_diffusion = DiffusionPipeline.from_pretrained('teticio/audio-diffusion-ddim-256').to(device)"
+    "audio_diffusion = DiffusionPipeline.from_pretrained('teticio/audio-diffusion-ddim-256-new').to(device)\n",
+    "mel = audio_diffusion.mel\n",
+    "sample_rate = mel.get_sample_rate()"
    ]
   },
   {
@@ -381,7 +380,7 @@
     "    seed = generator.seed()\n",
     "    print(f'Seed = {seed}')\n",
     "    generator.manual_seed(seed)\n",
-    "    output = audio_diffusion(mel=mel, generator=generator)\n",
+    "    output = audio_diffusion(generator=generator)\n",
     "    image = output.images[0]\n",
     "    audio = output.audios[0, 0]\n",
     "    display(image)\n",
@@ -410,7 +409,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "output = audio_diffusion(mel=mel, steps=1000, generator=generator, eta=1)\n",
+    "output = audio_diffusion(steps=1000, generator=generator, eta=1)\n",
     "image = output.images[0]\n",
     "audio = output.audios[0, 0]\n",
     "display(image)\n",
@@ -509,7 +508,6 @@
    "source": [
     "alpha = 0.5  #@param {type:\"slider\", min:0, max:1, step:0.1}\n",
     "output = audio_diffusion(\n",
-    "    mel=mel,\n",
     "    noise=audio_diffusion.slerp(noise, noise2, alpha),\n",
     "    generator=generator)\n",
     "audio = output.audios[0, 0]\n",
@@ -534,7 +532,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "model_id = \"teticio/latent-audio-diffusion-ddim-256\"  #@param [\"teticio/latent-audio-diffusion-256\", \"teticio/latent-audio-diffusion-ddim-256\"]"
+    "model_id = \"teticio/latent-audio-diffusion-ddim-256-new\"  #@param [\"teticio/latent-audio-diffusion-256\", \"teticio/latent-audio-diffusion-ddim-256\"]"
    ]
   },
   {
@@ -544,7 +542,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "audio_diffusion = DiffusionPipeline.from_pretrained(model_id).to(device)"
+    "audio_diffusion = DiffusionPipeline.from_pretrained(model_id).to(device)\n",
+    "mel = audio_diffusion.mel\n",
+    "sample_rate = mel.get_sample_rate()"
    ]
   },
   {
@@ -556,7 +556,7 @@
    "source": [
     "seed = 3412253600050855  #@param {type:\"integer\"}\n",
     "generator.manual_seed(seed)\n",
-    "output = audio_diffusion(mel=mel, generator=generator)\n",
+    "output = audio_diffusion(generator=generator)\n",
     "image = output.images[0]\n",
     "audio = output.audios[0, 0]\n",
     "display(image)\n",
@@ -572,7 +572,7 @@
    "source": [
     "seed2 = 7016114633369557  #@param {type:\"integer\"}\n",
     "generator.manual_seed(seed2)\n",
-    "output = audio_diffusion(mel=mel, generator=generator)\n",
+    "output = audio_diffusion(generator=generator)\n",
     "image2 = output.images[0]\n",
     "audio2 = output.audios[0, 0]\n",
     "display(image2)\n",
@@ -628,7 +628,6 @@
    "source": [
     "alpha = 0.5  #@param {type:\"slider\", min:0, max:1, step:0.1}\n",
     "output = audio_diffusion(\n",
-    "    mel=mel,\n",
     "    noise=audio_diffusion.slerp(latents, latents2, alpha),\n",
     "    generator=generator)\n",
     "audio3 = output.audios[0, 0]\n",