do not use replicate, run model locally

main.py

@@ -1,4 +1,7 @@
-import replicate
+import torch
+import requests
+from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
+from datasets import load_dataset
 from pydantic import BaseModel
 from fastapi import FastAPI
 
@@ -9,12 +12,36 @@ class URLPayload(BaseModel):
 app = FastAPI()
 
 def process_audio(url: str):
-    deployment = replicate.deployments.get("meal/incredibly-fast-whisper")
-    prediction = deployment.predictions.create(
-            input={ "audio": url }
+    response = requests.get(url)
+    with open("audio.mp3", mode="wb") as file:
+        file.write(response.content)
+
+
+    device = "cpu"
+
+    model_id = "openai/whisper-large-v3"
+    model = AutoModelForSpeechSeq2Seq.from_pretrained(
+            model_id, torch_dtype=torch.float32, low_cpu_mem_usage=True, use_safetensors=True
             )
-    prediction.wait()
-    return prediction.output
+    model.to(device)
+
+    processor = AutoProcessor.from_pretrained(model_id)
+    pipe = pipeline(
+            "automatic-speech-recognition",
+            model=model,
+            tokenizer=processor.tokenizer,
+            feature_extractor=processor.feature_extractor,
+            max_new_tokens=8192,
+            chunk_length_s=30,
+            batch_size=16,
+            return_timestamps=True,
+            torch_dtype=torch.float32,
+            device=device
+    )
+    dataset = load_dataset("distil-whisper/librispeech_long", "clean", split="validation")
+    whisper_result = pipe("audio.mp3")
+    return whisper_result
+
 
 @app.post("/process/")
 async def process_audio_endpoint(payload: URLPayload):

requirements.txt

@@ -1,21 +1,88 @@
+accelerate==0.26.1
+aiohttp==3.9.1
+aiosignal==1.3.1
 annotated-types==0.6.0
 anyio==4.2.0
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
 certifi==2023.11.17
+cffi==1.16.0
+charset-normalizer==3.3.2
 click==8.1.7
+datasets==2.16.1
+decorator==5.1.1
+dill==0.3.7
 distro==1.9.0
 exceptiongroup==1.2.0
 fastapi==0.108.0
+filelock==3.13.1
+frozenlist==1.4.1
+fsspec==2023.10.0
 h11==0.14.0
 httpcore==1.0.2
 httpx==0.26.0
+huggingface-hub==0.20.2
 idna==3.6
+Jinja2==3.1.3
+joblib==1.3.2
+lazy_loader==0.3
+librosa==0.10.1
+llvmlite==0.41.1
+MarkupSafe==2.1.3
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+multiprocess==0.70.15
+networkx==3.2.1
+numba==0.58.1
+numpy==1.26.3
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.18.1
+nvidia-nvjitlink-cu12==12.3.101
+nvidia-nvtx-cu12==12.1.105
 openai==1.7.1
 packaging==23.2
+pandas==2.1.4
+platformdirs==4.1.0
+pooch==1.8.0
+psutil==5.9.7
+pyarrow==14.0.2
+pyarrow-hotfix==0.6
+pycparser==2.21
 pydantic==2.5.3
 pydantic_core==2.14.6
-replicate==0.22.0
+python-dateutil==2.8.2
+pytz==2023.3.post1
+PyYAML==6.0.1
+regex==2023.12.25
+requests==2.31.0
+safetensors==0.4.1
+scikit-learn==1.3.2
+scipy==1.11.4
+six==1.16.0
 sniffio==1.3.0
+soundfile==0.12.1
+soxr==0.3.7
 starlette==0.32.0.post1
+sympy==1.12
+threadpoolctl==3.2.0
+tokenizers==0.15.0
+torch==2.1.2
 tqdm==4.66.1
+transformers @ git+https://github.com/huggingface/transformers.git@64bdbd888c78dcef5aeaeabc842e12981c8aae7a
+triton==2.1.0
 typing_extensions==4.9.0
+tzdata==2023.4
+urllib3==2.1.0
 uvicorn==0.25.0
+xxhash==3.4.1
+yarl==1.9.4