import os
import json
import uuid
import time
from fastapi import FastAPI, Request
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse, JSONResponse
from pydantic import BaseModel
from typing import List, Optional
import uvicorn
import torch
try:
from unsloth import FastLanguageModel
UNSLOTH_AVAILABLE = True
except ImportError:
UNSLOTH_AVAILABLE = False
# ──────────────────────────────────────────────────────────
# CONFIGURATION
# ──────────────────────────────────────────────────────────
DATASET_FILE = os.path.join("datasets", "human_alignment.jsonl")
os.makedirs("datasets", exist_ok=True)
# ──────────────────────────────────────────────────────────
# FASTAPI INIT
# ──────────────────────────────────────────────────────────
app = FastAPI(title="SAIL Sovereign Chat")
# Data Models
class Message(BaseModel):
role: str
content: str
class ChatRequest(BaseModel):
messages: List[Message]
class FeedbackRequest(BaseModel):
conversation_id: str
user_prompt: str
ai_response: str
feedback: str # "good" or "bad"
# ──────────────────────────────────────────────────────────
# MODEL INFERENCE CORE (UNSLOTH NATIVE)
# ──────────────────────────────────────────────────────────
MODEL_PATH = os.path.join(os.path.dirname(__file__), "sail_5b_hf_model")
model = None
tokenizer = None
def load_model():
global model, tokenizer
if model is not None:
return
print("--------------------------------------------------")
print("🧠 BOOTING 5-BILLION PARAMETER ENGINE INTO VRAM...")
print("Loading via Unsloth 4-bit NF4 to secure 8GB RTX 4060 Space.")
print("--------------------------------------------------")
if UNSLOTH_AVAILABLE:
model, tokenizer = FastLanguageModel.from_pretrained(
model_name=MODEL_PATH,
max_seq_length=2048,
load_in_4bit=True,
fast_inference=True,
)
FastLanguageModel.for_inference(model) # Enable native 2x inference speed
print("✅ 5B Engine Locked and Loaded Successfully.")
else:
print("❌ CRITICAL: Unsloth not found. Model cannot fit in VRAM natively.")
# Initialize the model right at boot as requested!
if os.path.exists(MODEL_PATH) and UNSLOTH_AVAILABLE:
load_model()
def generate_reply(conv_history: list) -> str:
if model is None or tokenizer is None:
return "\nThe 5B weights haven't been loaded into memory correctly.\n\n\nError: Model architecture is not loaded. Make sure `sail_5b_hf_model` exists and Unsloth is active."
# 1. Format the conversation for the LLM
try:
# If the tokenizer has a chat template, use it
prompt = tokenizer.apply_chat_template(
conv_history, tokenize=False, add_generation_prompt=True
)
except:
# Fallback raw formatting
prompt = ""
for msg in conv_history:
prompt += f"{msg['role'].capitalize()}: {msg['content']}\n"
prompt += "Assistant: "
# 2. Tokenize and Generate
inputs = tokenizer([prompt], return_tensors="pt").to("cuda")
# Generate 512 tokens
outputs = model.generate(
**inputs,
max_new_tokens=512,
use_cache=True,
temperature=0.7,
do_sample=True,
pad_token_id=tokenizer.eos_token_id
)
# 3. Decode exclusively the new tokens
generated_ids = outputs[0][inputs.input_ids.shape[1]:]
response_text = tokenizer.decode(generated_ids, skip_special_tokens=True).strip()
return response_text
def save_to_dataset(record: dict):
with open(DATASET_FILE, "a", encoding="utf-8") as f:
f.write(json.dumps(record) + "\n")
# ──────────────────────────────────────────────────────────
# ROUTES
# ──────────────────────────────────────────────────────────
@app.post("/api/chat")
async def api_chat(req: ChatRequest):
req_msgs = [{"role": m.role, "content": m.content} for m in req.messages]
# Generate reply natively using the 5B parameter weights in VRAM!
response_text = generate_reply(req_msgs)
conv_id = str(uuid.uuid4())
return JSONResponse({
"conversation_id": conv_id,
"reply": response_text
})
@app.post("/api/feedback")
async def api_feedback(req: FeedbackRequest):
# Save the RLHF preference to the dataset!
record = {
"id": req.conversation_id,
"prompt": req.user_prompt,
"chosen": req.ai_response if req.feedback == "good" else "",
"rejected": req.ai_response if req.feedback == "bad" else "",
"timestamp": time.time()
}
save_to_dataset(record)
return JSONResponse({"status": "saved"})
# ──────────────────────────────────────────────────────────
# SERVE FRONTEND (APPLE STYLE HTML/JS)
# ──────────────────────────────────────────────────────────
UI_PATH = os.path.join(os.path.dirname(__file__), "ui")
os.makedirs(UI_PATH, exist_ok=True)
# Mount the static files directory
app.mount("/static", StaticFiles(directory=UI_PATH), name="static")
@app.get("/")
async def root():
index_path = os.path.join(UI_PATH, "index.html")
if not os.path.exists(index_path):
return JSONResponse({"error": "UI files not built yet."})
return FileResponse(index_path)
if __name__ == "__main__":
print("--------------------------------------------------")
print("🚀 SAIL REST SERVER BOOTING...")
print("📡 Running on: http://127.0.0.1:8000")
print("--------------------------------------------------")
uvicorn.run("api_server:app", host="127.0.0.1", port=8000, reload=True)