push up batching for transcription (A10 GPU)

handler.py

@@ -31,7 +31,7 @@ SAMPLE_RATE = 16000
 def whisper_config():
     device = "cuda" if torch.cuda.is_available() else "cpu"
     whisper_model = "large-v3"
-    batch_size = 40 if device == "cuda" else 1
+    batch_size = 48 if device == "cuda" else 1
     compute_type = "float16" if device == "cuda" else "int8"
     return device, batch_size, compute_type, whisper_model
 

test endpoint.ipynb

@@ -2,13 +2,13 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
     "from pathlib import Path\n",
     "from retry import retry\n",
-    "import base64, requests, ffmpeg  # ffmpeg-python\n",
+    "import time, base64, requests, ffmpeg  # ffmpeg-python\n",
     "\n",
     "token = \"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\"\n",
     "API_URL = \"https://t4vtvikeag4f1yzd.eu-west-1.aws.endpoints.huggingface.cloud\"\n",
@@ -100,6 +100,7 @@
     "else:\n",
     "    print(\"Transcription first...\")\n",
     "    response = query_transcription(audio_data=audio_data)\n",
+    "    time.sleep(30)\n",
     "    print(\"...then Diarization.\")\n",
     "    response = query_diarization(audio_data=audio_data, transcription=response)"
    ]