initial commit

Dockerfile

@@ -0,0 +1,11 @@
+
+FROM python:3.10
+
+WORKDIR /code
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

app/main.py

@@ -0,0 +1,18 @@
+
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+from app.model_loader import load_model
+import torch
+
+app = FastAPI()
+model, tokenizer = load_model()
+
+@app.post("/predict")
+async def predict(request: Request):
+    data = await request.json()
+    input_text = data.get("input", "")
+    inputs = tokenizer(input_text, return_tensors="pt")
+    with torch.no_grad():
+        outputs = model.generate(**inputs, max_new_tokens=50)
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return JSONResponse(content={"output": response})

app/model_loader.py

@@ -0,0 +1,32 @@
+
+import os
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+
+os.environ["TRANSFORMERS_CACHE"] = "/tmp/hf_cache"
+os.makedirs("/tmp/hf_cache", exist_ok=True)
+
+def load_model():
+    hf_token = os.getenv("HF_TOKEN")
+    if not hf_token:
+        raise RuntimeError("HF_TOKEN not set.")
+
+    base_model = AutoModelForCausalLM.from_pretrained(
+        "meta-llama/Llama-2-7b-chat-hf",
+        use_auth_token=hf_token,
+        cache_dir="/tmp/hf_cache",
+        torch_dtype="auto",
+        device_map="auto"
+    )
+    model = PeftModel.from_pretrained(
+        base_model,
+        "BrainGPT/BrainGPT-7B-v0.1",
+        use_auth_token=hf_token,
+        cache_dir="/tmp/hf_cache"
+    )
+    tokenizer = AutoTokenizer.from_pretrained(
+        "meta-llama/Llama-2-7b-chat-hf",
+        use_auth_token=hf_token,
+        cache_dir="/tmp/hf_cache"
+    )
+    return model, tokenizer

requirements.txt

@@ -0,0 +1,7 @@
+
+transformers
+peft
+torch
+accelerate
+fastapi
+uvicorn