inside-out / app.py
peteryej's picture
auto-serve local llama models
ff730f4
Raw
History Blame Contribute Delete
21.9 kB
"""Inside Out - a chat where your emotions chime in.
A warm little app, inspired by Pixar's Inside Out, where several emotion agents
react to whatever is on your mind. The goal isn't to give advice - it's to help
you notice and name what you're actually feeling.
Run: python app.py
"""
from __future__ import annotations
import atexit
import html
import os
import re
import subprocess
import time
import urllib.request
from functools import partial
from urllib.parse import urlparse
from authlib.integrations.starlette_client import OAuth
from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse, RedirectResponse
import gradio as gr
from huggingface_hub import InferenceClient
from starlette.middleware.sessions import SessionMiddleware
import uvicorn
from agents import emotion_reply, run_turn
from emotions import EMOTIONS, EMOTION_ORDER
# Load variables from a local .env file if python-dotenv is available, so that
# HF_TOKEN, Google OAuth creds, etc. don't have to be exported by hand. Real
# shell environment variables still take precedence.
try:
from dotenv import load_dotenv
load_dotenv()
except ImportError:
pass
GOOGLE_CLIENT_ID = os.environ.get("GOOGLE_CLIENT_ID")
GOOGLE_CLIENT_SECRET = os.environ.get("GOOGLE_CLIENT_SECRET")
GOOGLE_ALLOWED_DOMAIN = os.environ.get("GOOGLE_ALLOWED_DOMAIN", "").lstrip("@")
GOOGLE_AUTH_ENABLED = bool(GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET)
SESSION_SECRET = os.environ.get("SESSION_SECRET", "inside-out-dev-session-secret")
HF_MODEL = "google/gemma-4-26B-A4B-it:deepinfra" # "openai/gpt-oss-20b", "Qwen/Qwen3.6-27B:ovhcloud"
SHOW_LOGIN = os.environ.get("SHOW_LOGIN", "").lower() in {"1", "true", "yes"}
# When True, talk to a local llama.cpp server (OpenAI-compatible API) instead of
# the hosted Hugging Face Inference API. Defaults to False; override with the
# env var LOCAL_SERVING=true. Start the server with, e.g.:
# llama-server -hf ggml-org/gemma-4-26b-a4b-it-GGUF:Q4_K_M \
# --jinja --reasoning-budget 0 -ngl 99 -c 8192 --port 8080
# NOTE: --reasoning-budget 0 is required for Gemma 4 - without it the model
# stays in "thinking" mode and returns empty content (replies fall back to
# canned demo lines). If port 8080 is taken, use another port and set
# LOCAL_LLM_BASE_URL to match (e.g. http://localhost:8088/v1).
LOCAL_SERVING = os.environ.get("LOCAL_SERVING", "false").lower() in {"1", "true", "yes"}
LOCAL_LLM_BASE_URL = os.environ.get("LOCAL_LLM_BASE_URL", "http://localhost:8088/v1")
LOCAL_LLM_MODEL = os.environ.get("LOCAL_LLM_MODEL", "nemotron-3-nano-omni-30b-a3b-reasoning") # "gemma-4-26b-a4b-it"
# Whether the served model is a reasoning/"thinking" model. When False (default)
# and we serve locally, llama.cpp is started with --reasoning-budget 0 so the
# model answers directly. When True, the model is allowed to think and the agent
# layer strips the chain-of-thought from replies. Override with USE_REASONING=true.
USE_REASONING = os.environ.get("USE_REASONING", "false").lower() in {"1", "true", "yes"}
# Where to look for local GGUF weights, and the llama.cpp server binary, used
# when LOCAL_SERVING auto-serves the model. LOCAL_MODEL_PATH pins an exact GGUF.
LOCAL_MODELS_DIR = os.environ.get("LOCAL_MODELS_DIR", os.path.expanduser("~/models"))
LOCAL_MODEL_PATH = os.environ.get("LOCAL_MODEL_PATH", "")
LLAMA_SERVER_BIN = os.environ.get(
"LLAMA_SERVER_BIN", os.path.expanduser("~/tools/llama.cpp/build/bin/llama-server")
)
# Startup diagnostic (boolean only - never logs the secret value) so you can
# confirm from the Space logs whether the HF_TOKEN secret reached the app.
print(
f"[inside-out] startup: serving="
f"{'local-llama.cpp' if LOCAL_SERVING else 'hf-inference'} "
f"| reasoning={'on' if USE_REASONING else 'off'} "
f"| HF_TOKEN present: {bool(os.environ.get('HF_TOKEN'))} "
f"| model: {LOCAL_LLM_MODEL if LOCAL_SERVING else HF_MODEL}",
flush=True,
)
oauth = OAuth()
if GOOGLE_AUTH_ENABLED:
oauth.register(
name="google",
client_id=GOOGLE_CLIENT_ID,
client_secret=GOOGLE_CLIENT_SECRET,
server_metadata_url="https://accounts.google.com/.well-known/openid-configuration",
client_kwargs={"scope": "openid email profile"},
)
THEME = gr.themes.Soft(
primary_hue="purple",
secondary_hue="yellow",
neutral_hue="slate",
font=[gr.themes.GoogleFont("Quicksand"), "ui-rounded", "system-ui", "sans-serif"],
)
CSS = """
.gradio-container {
background: radial-gradient(1200px 600px at 20% -10%, #fef3c7 0%, transparent 55%),
radial-gradient(1000px 700px at 90% 0%, #e9d5ff 0%, transparent 50%),
linear-gradient(160deg, #fdfbff 0%, #f3f0ff 100%) !important;
}
#title-wrap { text-align: center; margin: 6px 0 2px; }
#title-wrap h1 {
font-size: 2.5rem; font-weight: 700; margin-bottom: 2px;
background: linear-gradient(90deg,#f59e42,#f0533b,#9b6dd6,#5b8def,#ffd93b);
-webkit-background-clip: text; background-clip: text; color: transparent;
}
#subtitle { text-align:center; color:#6b6480; font-size:1.02rem; margin-bottom:10px; }
#legend-hint { text-align:center; color:#9a93ad; font-size:0.82rem; margin:2px 0 6px; }
/* Emotion chips are now clickable buttons. */
#legend {
display:flex !important; flex-wrap:wrap; gap:8px;
justify-content:center; margin:4px 0 14px;
background:transparent !important; border:0 !important;
}
#legend .emo-btn {
flex:0 0 auto !important; width:auto !important; min-width:0 !important;
padding:6px 14px !important; border-radius:999px !important;
font-size:0.86rem !important; font-weight:600 !important;
color:#3a3550 !important; background:#ffffffcc !important;
border:1.5px solid #d7cdef !important;
box-shadow:0 2px 8px rgba(120,100,180,0.10) !important;
backdrop-filter: blur(4px);
transition: transform .12s ease, box-shadow .12s ease;
}
#legend .emo-btn:hover {
transform: translateY(-1px);
box-shadow:0 5px 14px rgba(120,100,180,0.22) !important;
}
/* --- Chat surface: no grey boxes, let the bubbles breathe --- */
.gr-chatbot, #chat {
border:none !important;
background:transparent !important;
box-shadow:none !important;
}
/* Neutralise the grey bubble fill at the source: whichever wrapper class
this gradio version uses, none of them can paint the grey anymore. */
#chat, #chat * { --background-fill-secondary: transparent !important; }
#chat .bubble-wrap,
#chat .bot-row,
#chat .user-row,
#chat .message-wrap,
#chat .panel,
#chat .bubble { background: transparent !important; }
#chat .message-row { padding: 4px 0 !important; }
/* Bot / assistant messages: the colored inner cards stand on their own */
#chat .bot,
#chat .message.bot,
#chat .bubble,
#chat .bot .message-content,
#chat .message,
#chat .message-content {
background: transparent !important;
border: 0 !important;
box-shadow: none !important;
padding: 0 !important;
}
/* User messages: a soft glowing purple bubble */
#chat .user,
#chat .message.user {
background: linear-gradient(135deg, #a78bfa 0%, #8b5cf6 100%) !important;
color: #fff !important;
border: 0 !important;
border-radius: 16px 16px 4px 16px !important;
box-shadow: 0 6px 18px rgba(139,92,246,0.30) !important;
padding: 9px 14px !important;
max-width: 78%;
}
#chat .user *,
#chat .message.user * { color: #fff !important; background: transparent !important; }
#chat button[aria-label*="copy" i],
#chat button[aria-label*="clear" i] { display: none !important; }
/* --- Composer: a floating rounded pill --- */
#composer {
gap: 8px !important;
background: #ffffffcc;
border: 1.5px solid #e7dcff;
border-radius: 999px;
padding: 6px 8px 6px 18px;
box-shadow: 0 8px 24px rgba(140,110,210,0.12);
backdrop-filter: blur(6px);
align-items: center;
}
#composer textarea, #composer input[type="text"] {
background: transparent !important;
border: 0 !important;
box-shadow: none !important;
font-size: 1rem !important;
resize: none !important;
}
#composer button {
border-radius: 999px !important;
border: 0 !important;
background: linear-gradient(135deg, #f59e42, #f0533b) !important;
color: #fff !important;
font-weight: 700 !important;
box-shadow: 0 4px 12px rgba(240,83,59,0.30) !important;
}
footer { display:none !important; }
#reflect-note { color:#8a83a0; font-size:0.85rem; text-align:center; margin-top:6px; }
"""
# Tint each emotion chip-button with its own color.
CSS += "".join(
f"#emo-btn-{k} {{ border-color:{EMOTIONS[k].color} !important; "
f"color:{EMOTIONS[k].color} !important; }}\n"
f"#emo-btn-{k}:hover {{ background:{EMOTIONS[k].color}14 !important; }}\n"
for k in EMOTION_ORDER
)
def _safe_html(text: str) -> str:
return html.escape(text).replace("\n", "<br>")
def bubble(emo_key: str, text: str) -> dict:
"""Render one emotion's line as a styled assistant message."""
e = EMOTIONS[emo_key]
safe_text = _safe_html(text)
content = (
f'<div style="border-left:5px solid {e.color}; background:{e.color}1A; '
f'padding:9px 13px; border-radius:12px; margin:2px 0;">'
f'<div style="font-weight:700; color:{e.color}; font-size:0.9rem; '
f'margin-bottom:2px;">{e.emoji} {e.name}</div>'
f'<div style="color:#2f2b40; line-height:1.45;">{safe_text}</div></div>'
)
return {"role": "assistant", "content": content}
def reflect_bubble(text: str, allow_html: bool = False) -> dict:
rendered_text = text if allow_html else _safe_html(text)
content = (
'<div style="background:linear-gradient(135deg,#ffffff,#f6f1ff); '
'border:1px dashed #c7b8f0; padding:11px 15px; border-radius:14px; '
'margin:6px 0 2px; box-shadow:0 3px 14px rgba(150,120,220,0.12);">'
'<div style="font-weight:700; color:#8b5cf6; font-size:0.86rem; '
'margin-bottom:3px;">A gentle reflection</div>'
f'<div style="color:#3a3550; line-height:1.5;">{rendered_text}</div></div>'
)
return {"role": "assistant", "content": content}
def find_local_gguf(model_id: str) -> str | None:
"""Find a local .gguf for a model, or None if not available locally.
LOCAL_MODEL_PATH wins if set. Otherwise we walk LOCAL_MODELS_DIR and pick the
file whose name shares the most tokens with the model id (so e.g.
'nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16' matches a local
'NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-*.gguf').
"""
if LOCAL_MODEL_PATH:
return LOCAL_MODEL_PATH if os.path.isfile(LOCAL_MODEL_PATH) else None
if not os.path.isdir(LOCAL_MODELS_DIR):
return None
name = model_id.split(":", 1)[0].rsplit("/", 1)[-1].lower()
tokens = [t for t in re.split(r"[^a-z0-9]+", name) if t]
best, best_score = None, 0
for root, _dirs, files in os.walk(LOCAL_MODELS_DIR):
for fname in files:
low = fname.lower()
if not low.endswith(".gguf") or "mmproj" in low:
continue
score = sum(1 for t in tokens if t in low)
if score > best_score:
best, best_score = os.path.join(root, fname), score
# Require a meaningful overlap so we don't grab an unrelated model.
return best if best_score >= max(2, len(tokens) // 2) else None
_local_server_proc = None
def _server_healthy() -> bool:
health_url = LOCAL_LLM_BASE_URL.rsplit("/v1", 1)[0].rstrip("/") + "/health"
try:
with urllib.request.urlopen(health_url, timeout=2) as resp:
return resp.status == 200
except Exception:
return False
def ensure_local_server() -> None:
"""When LOCAL_SERVING, make sure a llama.cpp server is up for the model.
No-op if one is already responding (so uvicorn hot-reload won't double-spawn).
Launches llama-server with the locally-discovered GGUF, disabling the
reasoning budget unless USE_REASONING is set.
"""
global _local_server_proc
if not LOCAL_SERVING or _server_healthy():
return
gguf = find_local_gguf(HF_MODEL)
if not gguf:
print(
f"[inside-out] LOCAL_SERVING on but no local GGUF for {HF_MODEL!r} "
f"in {LOCAL_MODELS_DIR} (set LOCAL_MODEL_PATH). Falling back to demo.",
flush=True,
)
return
if not os.path.isfile(LLAMA_SERVER_BIN):
print(f"[inside-out] llama-server not found at {LLAMA_SERVER_BIN}", flush=True)
return
port = str(urlparse(LOCAL_LLM_BASE_URL).port or 8088)
cmd = [
LLAMA_SERVER_BIN, "-m", gguf, "--jinja",
"-ngl", "99", "-c", "8192", "--host", "127.0.0.1", "--port", port,
]
if not USE_REASONING:
cmd += ["--reasoning-budget", "0"]
print(
f"[inside-out] starting llama.cpp (reasoning={'on' if USE_REASONING else 'off'}): "
f"{os.path.basename(gguf)} on :{port}",
flush=True,
)
env = dict(os.environ, LD_LIBRARY_PATH=os.path.dirname(LLAMA_SERVER_BIN)
+ os.pathsep + os.environ.get("LD_LIBRARY_PATH", ""))
_local_server_proc = subprocess.Popen(cmd, env=env)
atexit.register(lambda: _local_server_proc and _local_server_proc.terminate())
for _ in range(240): # wait up to ~4 min for the weights to load
if _server_healthy():
print("[inside-out] llama.cpp server is ready", flush=True)
return
if _local_server_proc.poll() is not None:
print("[inside-out] llama.cpp server exited during startup", flush=True)
return
time.sleep(1)
print("[inside-out] timed out waiting for llama.cpp server", flush=True)
def _make_client():
"""Build the inference client.
- LOCAL_SERVING: ensure a local llama.cpp server is running and point at it
(OpenAI-compatible), no HF token needed.
- otherwise: the hosted HF Inference API, or None when no token is set
(which makes the app fall back to offline demo replies).
"""
if LOCAL_SERVING:
ensure_local_server()
return InferenceClient(
base_url=LOCAL_LLM_BASE_URL,
api_key=os.environ.get("LOCAL_LLM_API_KEY", "sk-no-key-needed"),
)
token = os.environ.get("HF_TOKEN")
return InferenceClient(token=token, model=HF_MODEL) if token else None
def _last_user_message(chat_history: list[dict]) -> str:
"""Most recent thing the person actually typed, for context."""
for turn in reversed(chat_history):
if turn.get("role") == "user":
return str(turn.get("content", "")).strip()
return ""
def respond(
message: str,
chat_history: list[dict] | None,
):
message = (message or "").strip()
chat_history = list(chat_history or [])
if not message:
return "", chat_history
client = _make_client()
chat_history = chat_history + [{"role": "user", "content": message}]
replies, reflection = run_turn(message, chat_history[:-1], client=client)
for key, text in replies:
chat_history.append(bubble(key, text))
chat_history.append(reflect_bubble(reflection))
return "", chat_history
def chime(emo_key: str, chat_history: list[dict] | None):
"""The user tapped an emotion chip - let that emotion speak up directly.
It reacts to the conversation so far (the most recent thing the person
said, plus recent history). No orchestrator, no reflection - just this one
emotion chiming in.
"""
chat_history = list(chat_history or [])
client = _make_client()
context = _last_user_message(chat_history)
if not context:
context = "They haven't said anything yet - gently invite them to share."
text = emotion_reply(emo_key, context, chat_history, client)
chat_history.append(bubble(emo_key, text))
return chat_history
def greeting() -> list[dict]:
text = (
"Hi there. This is a safe little space inside your head.<br>"
"Tell me what's on your mind - a thought, a moment, anything - and your "
"emotions will each chime in. There are no wrong feelings here."
)
return [reflect_bubble(text, allow_html=True)]
def create_demo(show_login: bool = False) -> gr.Blocks:
with gr.Blocks(title="Inside Out - Chat with your emotions") as blocks:
if show_login:
with gr.Sidebar():
if GOOGLE_AUTH_ENABLED:
gr.HTML(
'<a href="/logout" style="display:inline-block; padding:8px 12px; '
'border-radius:999px; border:1px solid #d8c7ff; '
'background:#ffffffcc; color:#6d28d9; font-weight:700; '
'text-decoration:none;">Sign out</a>'
)
else:
gr.Markdown(
"Google login is off locally. Set `GOOGLE_CLIENT_ID` and "
"`GOOGLE_CLIENT_SECRET` to require Google sign-in."
)
gr.Markdown(
f"Set `HF_TOKEN` to let `{HF_MODEL}` generate emotion responses. "
"Without it, the app uses demo replies."
)
gr.HTML(
'<div id="title-wrap"><h1>Inside Out</h1></div>'
'<div id="subtitle">Share what\'s on your mind, and let your emotions '
'chime in to help you discover how you really feel.</div>'
)
gr.HTML(
'<div id="legend-hint">Tap an emotion to let it speak up</div>'
)
emo_buttons: dict[str, gr.Button] = {}
with gr.Row(elem_id="legend"):
for k in EMOTION_ORDER:
e = EMOTIONS[k]
emo_buttons[k] = gr.Button(
f"{e.emoji} {e.name}",
elem_id=f"emo-btn-{k}",
elem_classes="emo-btn",
size="sm",
variant="secondary",
)
clear = gr.Button(
"Start fresh", size="sm", variant="secondary", elem_id="start-fresh"
)
chatbot = gr.Chatbot(
value=greeting(),
elem_id="chat",
height=460,
show_label=False,
sanitize_html=False,
buttons=[],
group_consecutive_messages=False,
)
with gr.Row(elem_id="composer"):
msg = gr.Textbox(
placeholder="What's on your mind today?",
show_label=False,
scale=8,
autofocus=True,
container=False,
)
send = gr.Button("Share", variant="primary", scale=1)
gr.HTML('<div id="reflect-note">Made with love.</div>')
send.click(respond, [msg, chatbot], [msg, chatbot])
msg.submit(respond, [msg, chatbot], [msg, chatbot])
clear.click(lambda: greeting(), None, chatbot)
for k, btn in emo_buttons.items():
btn.click(partial(chime, k), chatbot, chatbot)
return blocks
demo = create_demo(show_login=SHOW_LOGIN)
def _google_user(request: Request) -> str | None:
user = request.session.get("google_user")
if not isinstance(user, dict):
return None
email = user.get("email")
return str(email) if email else None
server = FastAPI()
@server.middleware("http")
async def redirect_unauthenticated_root(request: Request, call_next):
if GOOGLE_AUTH_ENABLED and request.method == "GET" and request.url.path == "/":
user = request.session.get("google_user")
if not user:
return RedirectResponse(url="/login")
return await call_next(request)
server.add_middleware(
SessionMiddleware,
secret_key=SESSION_SECRET,
same_site="lax",
https_only=os.environ.get("COOKIE_SECURE", "").lower() in {"1", "true", "yes"},
)
@server.get("/login")
async def google_login(request: Request):
if not GOOGLE_AUTH_ENABLED:
return HTMLResponse(
"<h1>Google login is not configured</h1>"
"<p>Set GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET, then restart the app.</p>"
'<p><a href="/">Continue to local demo</a></p>'
)
redirect_uri = request.url_for("google_auth_callback")
return await oauth.google.authorize_redirect(request, redirect_uri)
@server.get("/auth/callback")
async def google_auth_callback(request: Request):
token = await oauth.google.authorize_access_token(request)
userinfo = token.get("userinfo")
if userinfo is None:
userinfo = await oauth.google.userinfo(token=token)
email = userinfo.get("email")
if not email:
return HTMLResponse("<h1>Google login did not return an email address.</h1>", status_code=400)
domain = email.rsplit("@", 1)[-1]
if GOOGLE_ALLOWED_DOMAIN and domain != GOOGLE_ALLOWED_DOMAIN:
return HTMLResponse(
f"<h1>Access denied</h1><p>{html.escape(email)} is not in the allowed domain.</p>",
status_code=403,
)
request.session["google_user"] = {
"email": email,
"name": userinfo.get("name") or email,
"picture": userinfo.get("picture"),
}
return RedirectResponse(url="/")
@server.get("/logout")
async def google_logout(request: Request):
request.session.clear()
return RedirectResponse(url="/login" if GOOGLE_AUTH_ENABLED else "/")
app = gr.mount_gradio_app(
server,
demo,
path="/",
theme=THEME,
css=CSS,
auth_dependency=_google_user if GOOGLE_AUTH_ENABLED else None,
ssr_mode=False,
)
if __name__ == "__main__":
# Set DEV=1 to hot-reload on file changes (uvicorn watches the source and
# re-imports the module, so CSS, callbacks, and the FastAPI mount all refresh).
dev = os.environ.get("DEV", "").lower() in {"1", "true", "yes"}
uvicorn.run(
"app:app" if dev else app,
host="0.0.0.0",
port=int(os.environ.get("PORT", "7860")),
reload=dev,
)