Subh775 commited on
Commit
b0737b6
Β·
1 Parent(s): c5b24e6

dynamic routing for model selection

Browse files
Files changed (2) hide show
  1. app.py +4 -3
  2. graph.py +276 -130
app.py CHANGED
@@ -18,7 +18,7 @@ import guard
18
  import retriever
19
  import auth as auth_module
20
  from config import EXPORT_SECRET, REDIS_URI, GOOGLE_CLIENT_ID
21
- from graph import chatbot, TEXT_MODEL, VISION_MODEL
22
 
23
 
24
  app = FastAPI()
@@ -277,7 +277,7 @@ def chat(request: ChatRequest, req: Request):
277
  # Step 5: Stream LLM response via LangGraph
278
  yield sse_status("Preparing the explanation...")
279
  has_image = bool(request.image)
280
- model_used = VISION_MODEL if has_image else TEXT_MODEL
281
  config = {
282
  "configurable": {
283
  "thread_id": request.thread_id,
@@ -315,7 +315,8 @@ def chat(request: ChatRequest, req: Request):
315
  )
316
  elif "404" in str(e) or "not found" in error_str or "no endpoints" in error_str:
317
  yield sse_error(
318
- "The AI model is temporarily unavailable. Please try again in a moment."
 
319
  )
320
  elif "401" in str(e) or "unauthorized" in error_str or "invalid" in error_str:
321
  yield sse_error(
 
18
  import retriever
19
  import auth as auth_module
20
  from config import EXPORT_SECRET, REDIS_URI, GOOGLE_CLIENT_ID
21
+ from graph import chatbot
22
 
23
 
24
  app = FastAPI()
 
277
  # Step 5: Stream LLM response via LangGraph
278
  yield sse_status("Preparing the explanation...")
279
  has_image = bool(request.image)
280
+ model_used = "auto-routed"
281
  config = {
282
  "configurable": {
283
  "thread_id": request.thread_id,
 
315
  )
316
  elif "404" in str(e) or "not found" in error_str or "no endpoints" in error_str:
317
  yield sse_error(
318
+ "The selected model is temporarily unavailable. "
319
+ "Retrying may route to a different model. Please try again."
320
  )
321
  elif "401" in str(e) or "unauthorized" in error_str or "invalid" in error_str:
322
  yield sse_error(
graph.py CHANGED
@@ -1,12 +1,19 @@
1
  """
2
- LangGraph definition β€” single chat node with dynamic free OpenRouter model selection.
3
- Fetches available free models from OpenRouter API, caches for 10 minutes.
4
- Defaults to openai/gpt-oss-120b:free for text, nex-agi/nex-n2-pro:free for vision.
 
 
 
 
 
 
 
5
  """
6
 
7
  from langgraph.graph import StateGraph, START, END
8
  from langgraph.graph.message import add_messages
9
- from langchain_core.messages import BaseMessage, SystemMessage
10
  from langchain_core.runnables import RunnableConfig
11
  from langchain_openrouter import ChatOpenRouter
12
  from typing import TypedDict, Annotated
@@ -21,125 +28,250 @@ from config import OPENROUTER_API_KEY, DB_PATH
21
  import prompts
22
 
23
 
24
- # --- Checkpointer (SQLite β€” no external DB needed) ---
25
  _conn = sqlite3.connect(DB_PATH, check_same_thread=False)
26
  checkpointer = SqliteSaver(conn=_conn)
27
 
28
 
29
- # --- Dynamic free model selection ---
30
-
31
- DEFAULT_TEXT_MODEL = "openai/gpt-oss-120b:free"
32
- DEFAULT_VISION_MODEL = "nex-agi/nex-n2-pro:free"
33
-
34
- _models_cache = None
35
- _models_cache_at = 0
36
- _models_lock = threading.Lock()
37
- MODELS_TTL = 10 * 60 # 10 minutes
38
-
39
-
40
- def _fetch_free_models() -> list[dict]:
41
- """Fetch available free models from OpenRouter API."""
42
- global _models_cache, _models_cache_at
43
-
44
- with _models_lock:
45
- if _models_cache and (time.time() - _models_cache_at) < MODELS_TTL:
46
- return _models_cache
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
 
48
  try:
49
  req = urllib.request.Request(
50
  "https://openrouter.ai/api/v1/models",
51
- headers={"HTTP-Referer": "https://stemcopilot.app", "User-Agent": "STEMCopilot/1.0"},
 
 
 
52
  )
53
- with urllib.request.urlopen(req, timeout=8) as resp:
54
- data = json.loads(resp.read().decode())
55
-
56
- all_models = [
57
- {"id": m["id"], "name": m.get("name", m["id"])}
58
- for m in data.get("data", [])
59
- if m.get("id", "").endswith(":free")
60
- ]
61
-
62
- # Separate text and vision-capable models
63
- with _models_lock:
64
- _models_cache = all_models
65
- _models_cache_at = time.time()
66
-
67
- return all_models
68
-
69
- except Exception as e:
70
- print(f"[MODELS] Could not fetch free models: {e}")
71
- return _models_cache or []
72
-
73
-
74
- def _pick_text_model() -> str:
75
- """Pick the best free text model. Prefers GPT OSS 120B, then NVIDIA nemotron models."""
76
- models = _fetch_free_models()
77
- model_ids = {m["id"] for m in models}
78
-
79
- # Priority order
80
- preferred = [
81
- DEFAULT_TEXT_MODEL,
82
- "openai/gpt-oss-120b:free",
83
- ]
84
-
85
- # Check preferred first
86
- for mid in preferred:
87
- if mid in model_ids:
88
- return mid
89
-
90
- # Then any NVIDIA model
91
- nvidia = [m["id"] for m in models if m["id"].startswith("nvidia/")]
92
- if nvidia:
93
- return nvidia[0]
94
-
95
- # Then any free model that isn't tiny
96
- if models:
97
- return models[0]["id"]
98
-
99
- # Ultimate fallback
100
- return DEFAULT_TEXT_MODEL
101
-
102
-
103
- def _pick_vision_model() -> str:
104
- """Pick the best free vision/multimodal model."""
105
- models = _fetch_free_models()
106
- model_ids = {m["id"] for m in models}
107
-
108
- # Priority order for vision
109
- preferred = [
110
- DEFAULT_VISION_MODEL,
111
- "nex-agi/nex-n2-pro:free",
112
- ]
113
-
114
- for mid in preferred:
115
- if mid in model_ids:
116
- return mid
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
117
 
118
- # Fallback to text model β€” it may handle images poorly but won't 404
119
- return _pick_text_model()
120
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
121
 
122
- # Expose for app.py logging
123
- TEXT_MODEL = DEFAULT_TEXT_MODEL
124
- VISION_MODEL = DEFAULT_VISION_MODEL
125
 
 
126
 
127
- def _get_llm(api_key: str = "", model: str = "", has_image: bool = False):
128
- """Create an LLM instance. Dynamically picks the best available free model."""
129
  key = api_key or OPENROUTER_API_KEY
130
  if not key:
131
- raise ValueError("No API key available. Please add your OpenRouter key in Settings.")
132
-
133
- # Model priority: explicit override > dynamic selection
134
- if model:
135
- mdl = model
136
- elif has_image:
137
- mdl = _pick_vision_model()
138
- else:
139
- mdl = _pick_text_model()
140
-
141
  return ChatOpenRouter(
142
- model=mdl,
143
  openrouter_api_key=key,
144
  temperature=0.5,
145
  max_tokens=4096,
@@ -148,40 +280,54 @@ def _get_llm(api_key: str = "", model: str = "", has_image: bool = False):
148
  )
149
 
150
 
 
 
151
  class ChatState(TypedDict):
152
  messages: Annotated[list[BaseMessage], add_messages]
153
 
154
 
155
  def chat_node(state: ChatState, config: RunnableConfig):
156
- """
157
- Build system prompt from persona + language + context + user info,
158
- create per-request LLM with user's API key, invoke.
159
- """
160
- cfg = config.get("configurable", {})
161
- persona = cfg.get("persona", "nerd")
162
- context = cfg.get("context", "")
163
- language = cfg.get("language", "auto")
164
- username = cfg.get("username", "")
165
- student_profile = cfg.get("student_profile", "")
166
- user_api_key = cfg.get("user_api_key", "")
167
- model_name = cfg.get("model", "")
168
  has_image = cfg.get("has_image", False)
169
 
170
- system_msg = SystemMessage(
171
- content=prompts.build(persona, context, language, username, student_profile)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
172
  )
 
173
 
174
- llm = _get_llm(api_key=user_api_key, model=model_name, has_image=has_image)
175
- all_msgs = [system_msg] + state["messages"]
176
- response = llm.invoke(all_msgs)
177
 
178
- return {"messages": [response]}
179
 
 
180
 
181
- # --- Compile graph ---
182
- _graph = StateGraph(ChatState)
183
- _graph.add_node("chat_node", chat_node)
184
- _graph.add_edge(START, "chat_node")
185
- _graph.add_edge("chat_node", END)
186
 
187
- chatbot = _graph.compile(checkpointer=checkpointer)
 
1
  """
2
+ LangGraph β€” Intelligent Model Router for STEM Copilot.
3
+
4
+ Routes queries to the best free OpenRouter model based on intent:
5
+ - Math / derivation / proofs β†’ top reasoning models (rotated)
6
+ - Physics / chemistry concepts β†’ strong general models (rotated)
7
+ - Casual greetings ("hi", "ok") β†’ lightweight models
8
+ - Image understanding β†’ vision-capable models (verified)
9
+
10
+ Model pools are fetched from the OpenRouter API, cached for 10 min,
11
+ and rotated per-category to avoid per-model rate limits.
12
  """
13
 
14
  from langgraph.graph import StateGraph, START, END
15
  from langgraph.graph.message import add_messages
16
+ from langchain_core.messages import BaseMessage, SystemMessage, HumanMessage
17
  from langchain_core.runnables import RunnableConfig
18
  from langchain_openrouter import ChatOpenRouter
19
  from typing import TypedDict, Annotated
 
28
  import prompts
29
 
30
 
31
+ # ── Checkpointer ──────────────────────────────────────────────
32
  _conn = sqlite3.connect(DB_PATH, check_same_thread=False)
33
  checkpointer = SqliteSaver(conn=_conn)
34
 
35
 
36
+ # ── Constants ─────────────────────────────────────────────────
37
+ DEFAULT_MODEL = "openai/gpt-oss-120b:free"
38
+ MODELS_TTL = 10 * 60 # seconds
39
+
40
+
41
+ # ── Query Classification ─────────────────────────────────────
42
+
43
+ _CASUAL_EXACT = frozenset([
44
+ "hi", "hii", "hiii", "hello", "hey", "yo", "sup", "hola",
45
+ "thanks", "thank you", "thankyou", "thx", "ty",
46
+ "ok", "okay", "k", "kk", "fine", "alright",
47
+ "bye", "goodbye", "see you", "later",
48
+ "good morning", "good night", "good evening", "gm", "gn",
49
+ "welcome", "cool", "nice", "great", "awesome", "wow",
50
+ "got it", "understood", "sure", "yes", "no", "yeah", "nah", "yep", "nope",
51
+ "hm", "hmm", "oh", "ah", "lol", "haha", "hehe", "xd",
52
+ "what", "nothing", "nvm", "nevermind", "nm",
53
+ ])
54
+
55
+ _REASONING_KW = [
56
+ "derive", "derivation", "prove", "proof", "solve", "equation",
57
+ "integral", "integrate", "differentiate", "differentiation",
58
+ "formula", "calculate", "calculation", "compute",
59
+ "theorem", "lemma", "corollary",
60
+ "limit", "matrix", "determinant", "vector", "eigen",
61
+ "trigonometry", "trigonometric",
62
+ "quadratic", "polynomial", "logarithm",
63
+ "calculus", "algebra", "geometry", "coordinate",
64
+ "probability", "permutation", "combination",
65
+ "step by step", "show steps", "show working", "work out",
66
+ "simplify", "factorize", "factorise", "expand",
67
+ "numerator", "denominator", "fraction",
68
+ "dx", "dy", "dz", " lim ",
69
+ ]
70
+
71
+ _SCIENCE_KW = [
72
+ "physics", "chemistry", "biology",
73
+ "molecule", "atom", "electron", "proton", "neutron",
74
+ "ion", "isotope", "nucleus",
75
+ "force", "energy", "momentum", "velocity", "acceleration",
76
+ "wave", "wavelength", "frequency", "amplitude",
77
+ "light", "optics", "lens", "mirror", "refraction", "reflection",
78
+ "electric", "magnetic", "electromagnetic", "circuit", "resistance",
79
+ "gravity", "gravitational", "newton", "coulomb",
80
+ "reaction", "compound", "element", "bond", "orbital",
81
+ "hybridisation", "hybridization", "valence",
82
+ "thermodynamics", "entropy", "enthalpy",
83
+ "kinematics", "dynamics",
84
+ "quantum", "relativity", "nuclear",
85
+ "acid", "base", "salt", "buffer",
86
+ "oxidation", "reduction", "redox",
87
+ "mole", "molarity", "avogadro",
88
+ "ncert", "class 11", "class 12", "class xi", "class xii",
89
+ ]
90
+
91
+
92
+ def _classify(text: str) -> str:
93
+ """Return one of: casual, reasoning, science, general."""
94
+ t = text.strip().lower()
95
+ if t in _CASUAL_EXACT or len(t) < 6:
96
+ return "casual"
97
+ if any(kw in t for kw in _REASONING_KW):
98
+ return "reasoning"
99
+ if any(kw in t for kw in _SCIENCE_KW):
100
+ return "science"
101
+ return "general"
102
+
103
+
104
+ # ── Model Scoring ──────────────────────���──────────────────────
105
+
106
+ def _score(model_id: str) -> int:
107
+ """Higher score β‰ˆ better for hard reasoning tasks."""
108
+ m = model_id.lower()
109
+ s = 50
110
+ for tag, pts in [
111
+ ("253b", 100), ("250b", 100), ("200b", 95),
112
+ ("120b", 90), ("110b", 88), ("100b", 85),
113
+ ("70b", 70), ("72b", 70), ("65b", 68),
114
+ ("49b", 55), ("46b", 55), ("40b", 55),
115
+ ("27b", 40), ("32b", 42), ("34b", 42),
116
+ ("13b", 20), ("14b", 20), ("8b", 15),
117
+ ("3b", -20), ("1b", -40), ("0.5b", -50),
118
+ ]:
119
+ if tag in m:
120
+ s += pts
121
+ break
122
+ if "nemotron" in m: s += 15
123
+ if "gpt" in m: s += 10
124
+ if "llama" in m: s += 5
125
+ if "qwen" in m: s += 5
126
+ if "deepseek" in m: s += 8
127
+ return s
128
+
129
+
130
+ # ── Model Pool Fetching & Caching ─────────────────────────────
131
+
132
+ _cache: dict | None = None
133
+ _cache_at = 0.0
134
+ _lock = threading.Lock()
135
+ _counters: dict[str, int] = {}
136
+
137
+
138
+ def _fetch_pools() -> dict:
139
+ """Fetch free models, detect vision support, build sorted pools."""
140
+ global _cache, _cache_at
141
+
142
+ with _lock:
143
+ if _cache and (time.time() - _cache_at) < MODELS_TTL:
144
+ return _cache
145
 
146
  try:
147
  req = urllib.request.Request(
148
  "https://openrouter.ai/api/v1/models",
149
+ headers={
150
+ "HTTP-Referer": "https://stemcopilot.app",
151
+ "User-Agent": "STEMCopilot/1.0",
152
+ },
153
  )
154
+ with urllib.request.urlopen(req, timeout=10) as resp:
155
+ raw = json.loads(resp.read().decode())
156
+
157
+ text_all, vision_all = [], []
158
+
159
+ for m in raw.get("data", []):
160
+ mid = m.get("id", "")
161
+ if not mid.endswith(":free"):
162
+ continue
163
+
164
+ arch = m.get("architecture", {})
165
+ modality = arch.get("modality", "")
166
+ input_mods = arch.get("input_modalities", [])
167
+ has_vision = ("image" in modality) or ("image" in input_mods)
168
+
169
+ entry = {"id": mid, "score": _score(mid), "vision": has_vision}
170
+ text_all.append(entry)
171
+ if has_vision:
172
+ vision_all.append(entry)
173
+
174
+ # Sort high→low for reasoning, low→high for casual
175
+ text_all.sort(key=lambda x: x["score"], reverse=True)
176
+ vision_all.sort(key=lambda x: x["score"], reverse=True)
177
+
178
+ pools = {
179
+ "reasoning": text_all[:6], # best 6
180
+ "science": text_all[:8], # best 8
181
+ "general": text_all[:10], # best 10
182
+ "casual": text_all[-5:][::-1] if len(text_all) >= 5 else text_all[:3],
183
+ "vision": vision_all, # all vision-capable
184
+ }
185
+
186
+ with _lock:
187
+ _cache = pools
188
+ _cache_at = time.time()
189
+
190
+ print(f"[ROUTER] {len(text_all)} free models, {len(vision_all)} vision-capable")
191
+ for cat in ("reasoning", "casual", "vision"):
192
+ ids = [e["id"] for e in pools[cat][:3]]
193
+ if ids:
194
+ print(f"[ROUTER] {cat}: {ids}")
195
+
196
+ return pools
197
+
198
+ except Exception as exc:
199
+ print(f"[ROUTER] Fetch failed: {exc}")
200
+ fallback_entry = {"id": DEFAULT_MODEL, "score": 50, "vision": False}
201
+ return _cache or {
202
+ k: [fallback_entry]
203
+ for k in ("reasoning", "science", "general", "casual", "vision")
204
+ }
205
+
206
+
207
+ # ── Model Picker (round-robin) ────────────────────────────────
208
+
209
+ def _pick(category: str, has_image: bool = False) -> tuple[str, str]:
210
+ """
211
+ Return (model_id, actual_category).
212
+ If has_image but no vision model exists, falls back to text category.
213
+ """
214
+ pools = _fetch_pools()
215
+
216
+ if has_image:
217
+ v = pools.get("vision", [])
218
+ if v:
219
+ idx = _counters.get("vision", 0) % len(v)
220
+ _counters["vision"] = idx + 1
221
+ return v[idx]["id"], "vision"
222
+ # No vision models β€” caller must strip images
223
+ # Fall through to text routing
224
+
225
+ pool = pools.get(category) or pools.get("general") or []
226
+ if not pool:
227
+ return DEFAULT_MODEL, category
228
+
229
+ idx = _counters.get(category, 0) % len(pool)
230
+ _counters[category] = idx + 1
231
+ return pool[idx]["id"], category
232
+
233
+
234
+ # ── Message helpers ────────────────────────────────────────────
235
+
236
+ def _extract_text(messages: list) -> str:
237
+ """Get raw text from the last user message (handles multimodal)."""
238
+ if not messages:
239
+ return ""
240
+ content = messages[-1].content if hasattr(messages[-1], "content") else ""
241
+ if isinstance(content, list):
242
+ return " ".join(
243
+ p.get("text", "") for p in content
244
+ if isinstance(p, dict) and p.get("type") == "text"
245
+ )
246
+ return str(content)
247
 
 
 
248
 
249
+ def _strip_images(messages: list) -> list:
250
+ """Return messages with image_url parts removed."""
251
+ out = []
252
+ for msg in messages:
253
+ if isinstance(msg.content, list):
254
+ text = " ".join(
255
+ p.get("text", "")
256
+ for p in msg.content
257
+ if isinstance(p, dict) and p.get("type") == "text"
258
+ ).strip()
259
+ out.append(HumanMessage(content=text) if text else msg)
260
+ else:
261
+ out.append(msg)
262
+ return out
263
 
 
 
 
264
 
265
+ # ── LLM factory ───────────────────────────────────────────────
266
 
267
+ def _make_llm(api_key: str, model_id: str):
 
268
  key = api_key or OPENROUTER_API_KEY
269
  if not key:
270
+ raise ValueError(
271
+ "No API key available. Please add your OpenRouter key in Settings."
272
+ )
 
 
 
 
 
 
 
273
  return ChatOpenRouter(
274
+ model=model_id,
275
  openrouter_api_key=key,
276
  temperature=0.5,
277
  max_tokens=4096,
 
280
  )
281
 
282
 
283
+ # ── LangGraph state & node ────────────────────────────────────
284
+
285
  class ChatState(TypedDict):
286
  messages: Annotated[list[BaseMessage], add_messages]
287
 
288
 
289
  def chat_node(state: ChatState, config: RunnableConfig):
290
+ cfg = config.get("configurable", {})
291
+ persona = cfg.get("persona", "nerd")
292
+ context = cfg.get("context", "")
293
+ language = cfg.get("language", "auto")
294
+ username = cfg.get("username", "")
295
+ profile = cfg.get("student_profile", "")
296
+ api_key = cfg.get("user_api_key", "")
297
+ override = cfg.get("model", "")
 
 
 
 
298
  has_image = cfg.get("has_image", False)
299
 
300
+ # 1) Classify
301
+ user_text = _extract_text(state["messages"])
302
+ category = _classify(user_text)
303
+
304
+ # 2) Pick model
305
+ if override:
306
+ model_id, actual = override, category
307
+ else:
308
+ model_id, actual = _pick(category, has_image=has_image)
309
+
310
+ # 3) If vision unavailable, strip images so the text model won't 404
311
+ messages = state["messages"]
312
+ if has_image and actual != "vision":
313
+ messages = _strip_images(messages)
314
+
315
+ # 4) Build system prompt & invoke
316
+ sys = SystemMessage(
317
+ content=prompts.build(persona, context, language, username, profile)
318
  )
319
+ print(f"[ROUTER] category={category} model={model_id} vision={actual == 'vision'}")
320
 
321
+ llm = _make_llm(api_key, model_id)
322
+ resp = llm.invoke([sys] + messages)
323
+ return {"messages": [resp]}
324
 
 
325
 
326
+ # ── Compile graph ─────────────────────────────────────────────
327
 
328
+ _g = StateGraph(ChatState)
329
+ _g.add_node("chat_node", chat_node)
330
+ _g.add_edge(START, "chat_node")
331
+ _g.add_edge("chat_node", END)
 
332
 
333
+ chatbot = _g.compile(checkpointer=checkpointer)