File size: 5,185 Bytes
d6c005e 70e0661 d6c005e 70e0661 d6c005e 70e0661 d6c005e d696fcb d6c005e d696fcb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 | from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from typing import Optional
import time
import uuid
from datetime import datetime
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse
import os
from services.inference import run_single_strategy, run_all_strategies
from services.attention import extract_attention
from services.embeddings import compute_similarity
# ── Creating the FastAPI app ─────────────────────────────────────────────────
app = FastAPI(
title='Inference Observatory API',
description="Local inference engine for all 10 decoding strategies",
version="1.0.0",
)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
class RunAllRequest(BaseModel):
prompt: str
max_tokens: Optional[int] = 150
beam_size: Optional[int] = 5
temperature: Optional[float] = 0.7
top_k: Optional[int] = 50
top_p: Optional[float] = 0.9
tktp_k: Optional[int] = 50
tktp_p: Optional[float] = 0.9
ttk_temp: Optional[float] = 0.7
ttk_k: Optional[int] = 50
ttp_temp: Optional[float] = 0.7
ttp_p: Optional[float] = 0.9
ttkp_temp: Optional[float] = 0.7
ttkp_k: Optional[int] = 50
ttkp_p: Optional[float] = 0.9
class RunStrategyRequest(BaseModel):
strategy: str
params: RunAllRequest
class AttentionRequest(BaseModel):
word: str
prompt: str
class SimilarityRequest(BaseModel):
texts: dict
# ── ENDPOINTS ──────────────────────────────────────────────────────────────
frontend_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'frontend')
)
@app.get("/")
async def serve_frontend():
return FileResponse(os.path.join(frontend_path, 'index.html'))
@app.get("/styles.css")
async def serve_css():
return FileResponse(os.path.join(frontend_path, 'styles.css'))
@app.get("/app.js")
async def serve_js():
return FileResponse(os.path.join(frontend_path, 'app.js'))
@app.get("/health")
async def health():
return {
"status": "healthy",
"timestamp": time.time()
}
@app.post("/api/run-strategy")
async def run_strategy_endpoint(request: RunStrategyRequest):
try:
start_time = time.time()
output = await run_single_strategy(
strategy=request.strategy,
params=request.params
)
output["total_time_ms"] = round((time.time() - start_time) * 1000)
return output
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/api/run-all")
async def run_all(request: RunAllRequest):
try:
start_time = time.time()
outputs = await run_all_strategies(
prompt=request.prompt,
max_tokens=request.max_tokens,
beam_size=request.beam_size,
top_k=request.top_k,
top_p=request.top_p,
temperature=request.temperature,
tktp_k=request.tktp_k,
tktp_p=request.tktp_p,
ttk_temp=request.ttk_temp,
ttk_k=request.ttk_k,
ttp_temp=request.ttp_temp,
ttp_p=request.ttp_p,
ttkp_temp=request.ttkp_temp,
ttkp_k=request.ttkp_k,
ttkp_p=request.ttkp_p,
)
return {
"outputs": outputs,
"run_id": str(uuid.uuid4()),
"timestamp": datetime.utcnow().isoformat(),
"total_time_ms": round((time.time() - start_time) * 1000)
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/api/attention")
async def attention(request: AttentionRequest):
try:
result = await extract_attention(
prompt=request.prompt,
word=request.word
)
return result
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/api/similarity")
async def similarity(request: SimilarityRequest):
try:
scores = await compute_similarity(request.texts)
return {"scores": scores}
except Exception as e:
import traceback
traceback.print_exc()
raise HTTPException(status_code=500, detail=str(e))
# ── Run the server ─────────────────────────────────────────────────────────
if __name__ == "__main__":
import uvicorn
print("Starting Inference Observatory backend...")
print("API docs available at: http://localhost:8000/docs")
print("Model: Qwen2.5-0.5B (loads on first request)")
uvicorn.run(
"main:app",
host="0.0.0.0",
port=7860,
reload=True
)
|